diff --git a/addrmgr/addrmanager.go b/addrmgr/addrmanager.go index a30a51376..62d336db9 100644 --- a/addrmgr/addrmanager.go +++ b/addrmgr/addrmanager.go @@ -724,7 +724,11 @@ func (a *AddrManager) Start() { // Start the address ticker to save addresses periodically. a.wg.Add(1) - spawn(a.addressHandler) + spawn(a.addressHandler, a.handlePanic) +} + +func (a *AddrManager) handlePanic() { + atomic.AddInt32(&a.shutdown, 1) } // Stop gracefully shuts down the address manager by stopping the main handler. diff --git a/addrmgr/log.go b/addrmgr/log.go index 0cb5e431f..0563af0dc 100644 --- a/addrmgr/log.go +++ b/addrmgr/log.go @@ -10,4 +10,4 @@ import ( ) var log, _ = logger.Get(logger.SubsystemTags.ADXR) -var spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) +var spawn = panics.GoroutineWrapperFuncWithPanicHandler(log) diff --git a/apiserver/database/log.go b/apiserver/database/log.go index 4ea90e649..cc2f20ff7 100644 --- a/apiserver/database/log.go +++ b/apiserver/database/log.go @@ -5,5 +5,5 @@ import "github.com/daglabs/btcd/apiserver/logger" var ( log = logger.Logger("DTBS") - spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(log) ) diff --git a/apiserver/jsonrpc/log.go b/apiserver/jsonrpc/log.go index f6a6267a2..3f4155726 100644 --- a/apiserver/jsonrpc/log.go +++ b/apiserver/jsonrpc/log.go @@ -8,9 +8,9 @@ import ( var ( log = logger.BackendLog.Logger("RPCC") - spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(log) ) func init() { - rpcclient.UseLogger(log, logger.BackendLog) + rpcclient.UseLogger(log) } diff --git a/apiserver/log.go b/apiserver/log.go index ec9032a0e..92e365af3 100644 --- a/apiserver/log.go +++ b/apiserver/log.go @@ -7,5 +7,5 @@ import ( var ( log = logger.Logger("APIS") - spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(log) ) diff --git a/apiserver/main.go b/apiserver/main.go index 1aee39288..cbae1d7be 100644 --- a/apiserver/main.go +++ b/apiserver/main.go @@ -10,7 +10,6 @@ import ( "github.com/daglabs/btcd/apiserver/database" "github.com/daglabs/btcd/apiserver/jsonrpc" "github.com/daglabs/btcd/apiserver/server" - "github.com/daglabs/btcd/logger" "github.com/daglabs/btcd/signal" "github.com/daglabs/btcd/util/panics" _ "github.com/golang-migrate/migrate/v4/database/mysql" @@ -19,7 +18,7 @@ import ( ) func main() { - defer panics.HandlePanic(log, logger.BackendLog, nil) + defer panics.HandlePanic(log, nil, nil) err := config.Parse() if err != nil { diff --git a/apiserver/mqtt/log.go b/apiserver/mqtt/log.go index aa044a5a6..ee4e2210b 100644 --- a/apiserver/mqtt/log.go +++ b/apiserver/mqtt/log.go @@ -5,5 +5,5 @@ import "github.com/daglabs/btcd/apiserver/logger" var ( log = logger.Logger("MQTT") - spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(log) ) diff --git a/apiserver/server/log.go b/apiserver/server/log.go index d33d27d37..9609d76b0 100644 --- a/apiserver/server/log.go +++ b/apiserver/server/log.go @@ -5,5 +5,5 @@ import "github.com/daglabs/btcd/apiserver/logger" var ( log = logger.Logger("REST") - spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(log) ) diff --git a/blockdag/indexers/log.go b/blockdag/indexers/log.go index 2e8d447bb..1f0b00b17 100644 --- a/blockdag/indexers/log.go +++ b/blockdag/indexers/log.go @@ -10,4 +10,4 @@ import ( ) var log, _ = logger.Get(logger.SubsystemTags.INDX) -var spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) +var spawn = panics.GoroutineWrapperFunc(log) diff --git a/blockdag/log.go b/blockdag/log.go index a0804ac35..14ba9d526 100644 --- a/blockdag/log.go +++ b/blockdag/log.go @@ -10,4 +10,4 @@ import ( ) var log, _ = logger.Get(logger.SubsystemTags.BDAG) -var spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) +var spawn = panics.GoroutineWrapperFunc(log) diff --git a/btcd.go b/btcd.go index b169fa1cf..ca19df6a0 100644 --- a/btcd.go +++ b/btcd.go @@ -6,7 +6,6 @@ package main import ( "fmt" - "github.com/daglabs/btcd/logger" "net" "net/http" _ "net/http/pprof" @@ -57,7 +56,7 @@ func btcdMain(serverChan chan<- *server.Server) error { return err } cfg = config.ActiveConfig() - defer panics.HandlePanic(btcdLog, logger.BackendLog, nil) + defer panics.HandlePanic(btcdLog, nil, nil) // Get a channel that will be closed when a shutdown signal has been // triggered either from an OS signal such as SIGINT (Ctrl+C) or from diff --git a/cmd/addblock/addblock.go b/cmd/addblock/addblock.go index 2464fa8fb..060138db4 100644 --- a/cmd/addblock/addblock.go +++ b/cmd/addblock/addblock.go @@ -72,7 +72,7 @@ func realMain() error { backendLogger := logs.NewBackend() defer os.Stdout.Sync() log = backendLogger.Logger("MAIN") - spawn = panics.GoroutineWrapperFunc(log, backendLogger) + spawn = panics.GoroutineWrapperFunc(log) // Load the block database. db, err := loadBlockDB() diff --git a/cmd/txgen/log.go b/cmd/txgen/log.go index 1bfdcf5d2..5b1cf082f 100644 --- a/cmd/txgen/log.go +++ b/cmd/txgen/log.go @@ -10,7 +10,7 @@ import ( var ( backendLog = logs.NewBackend() log = backendLog.Logger("TXGN") - spawn = panics.GoroutineWrapperFunc(log, backendLog) + spawn = panics.GoroutineWrapperFunc(log) ) func initLog(logFile, errLogFile string) { diff --git a/cmd/txgen/main.go b/cmd/txgen/main.go index f3d5bcb47..490ca3823 100644 --- a/cmd/txgen/main.go +++ b/cmd/txgen/main.go @@ -23,7 +23,7 @@ func privateKeyToP2pkhAddress(key *btcec.PrivateKey, net *dagconfig.Params) (uti } func main() { - defer panics.HandlePanic(log, backendLog, nil) + defer panics.HandlePanic(log, nil, nil) cfg, err := parseConfig() if err != nil { diff --git a/connmgr/connmanager.go b/connmgr/connmanager.go index b90789cb3..244ed0c5e 100644 --- a/connmgr/connmanager.go +++ b/connmgr/connmanager.go @@ -247,7 +247,7 @@ func (cm *ConnManager) handleFailedConn(c *ConnReq, err error) { cm.NewConnReq() }) } else { - spawn(cm.NewConnReq) + spawn(cm.NewConnReq, cm.handlePanic) } } } @@ -584,7 +584,7 @@ func (cm *ConnManager) Start() { log.Trace("Connection manager started") cm.wg.Add(1) - spawn(cm.connHandler) + spawn(cm.connHandler, cm.handlePanic) // Start all the listeners so long as the caller requested them and // provided a callback to be invoked when connections are accepted. @@ -596,7 +596,7 @@ func (cm *ConnManager) Start() { } for i := atomic.LoadUint64(&cm.connReqCount); i < uint64(cm.cfg.TargetOutbound); i++ { - spawn(cm.NewConnReq) + spawn(cm.NewConnReq, cm.handlePanic) } } @@ -605,6 +605,10 @@ func (cm *ConnManager) Wait() { cm.wg.Wait() } +func (cm *ConnManager) handlePanic() { + atomic.AddInt32(&cm.stop, 1) +} + // Stop gracefully shuts down the connection manager. func (cm *ConnManager) Stop() { if atomic.AddInt32(&cm.stop, 1) != 1 { diff --git a/connmgr/log.go b/connmgr/log.go index f14cc7c11..2c1094a6e 100644 --- a/connmgr/log.go +++ b/connmgr/log.go @@ -10,4 +10,4 @@ import ( ) var log, _ = logger.Get(logger.SubsystemTags.CMGR) -var spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) +var spawn = panics.GoroutineWrapperFuncWithPanicHandler(log) diff --git a/database/cmd/dbtool/main.go b/database/cmd/dbtool/main.go index ec576eea9..01c6d5bd4 100644 --- a/database/cmd/dbtool/main.go +++ b/database/cmd/dbtool/main.go @@ -67,7 +67,7 @@ func realMain() error { backendLogger := logs.NewBackend() defer os.Stdout.Sync() log = backendLogger.Logger("MAIN") - spawn = panics.GoroutineWrapperFunc(log, backendLogger) + spawn = panics.GoroutineWrapperFunc(log) dbLog, _ := logger.Get(logger.SubsystemTags.BCDB) dbLog.SetLevel(logs.LevelDebug) diff --git a/dnsseeder/dnsseed.go b/dnsseeder/dnsseed.go index b32bb340c..f1fdefcbf 100644 --- a/dnsseeder/dnsseed.go +++ b/dnsseeder/dnsseed.go @@ -156,7 +156,7 @@ func creep() { } func main() { - defer panics.HandlePanic(log, backendLog, nil) + defer panics.HandlePanic(log, nil, nil) cfg, err := loadConfig() if err != nil { fmt.Fprintf(os.Stderr, "loadConfig: %v\n", err) diff --git a/dnsseeder/log.go b/dnsseeder/log.go index 6acd05f9f..ab04e191e 100644 --- a/dnsseeder/log.go +++ b/dnsseeder/log.go @@ -10,7 +10,7 @@ import ( var ( backendLog = logs.NewBackend() log = backendLog.Logger("SEED") - spawn = panics.GoroutineWrapperFunc(log, backendLog) + spawn = panics.GoroutineWrapperFunc(log) ) func initLog(logFile, errLogFile string) { diff --git a/faucet/database/log.go b/faucet/database/log.go index 05343395a..23bfa46ff 100644 --- a/faucet/database/log.go +++ b/faucet/database/log.go @@ -5,5 +5,5 @@ import "github.com/daglabs/btcd/apiserver/logger" var ( log = logger.BackendLog.Logger("DTBS") - spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(log) ) diff --git a/faucet/log.go b/faucet/log.go index 749aca430..984065bd0 100644 --- a/faucet/log.go +++ b/faucet/log.go @@ -7,5 +7,5 @@ import ( var ( log = logger.BackendLog.Logger("FAUC") - spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(log) ) diff --git a/faucet/main.go b/faucet/main.go index 3c5d50d8b..4a061453a 100644 --- a/faucet/main.go +++ b/faucet/main.go @@ -12,7 +12,6 @@ import ( "github.com/pkg/errors" "os" - "github.com/daglabs/btcd/logger" "github.com/daglabs/btcd/signal" "github.com/daglabs/btcd/util/panics" _ "github.com/golang-migrate/migrate/v4/database/mysql" @@ -27,7 +26,7 @@ var ( ) func main() { - defer panics.HandlePanic(log, logger.BackendLog, nil) + defer panics.HandlePanic(log, nil, nil) err := config.Parse() if err != nil { diff --git a/httpserverutils/log.go b/httpserverutils/log.go index 402395134..22bceb00b 100644 --- a/httpserverutils/log.go +++ b/httpserverutils/log.go @@ -5,5 +5,5 @@ import "github.com/daglabs/btcd/apiserver/logger" var ( log = logger.BackendLog.Logger("UTIL") - spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(log) ) diff --git a/log.go b/log.go index 6a2d6ed45..af04d13ac 100644 --- a/log.go +++ b/log.go @@ -11,5 +11,5 @@ import ( ) var btcdLog, _ = logger.Get(logger.SubsystemTags.BTCD) -var spawn = panics.GoroutineWrapperFunc(btcdLog, logger.BackendLog) +var spawn = panics.GoroutineWrapperFunc(btcdLog) var srvrLog, _ = logger.Get(logger.SubsystemTags.SRVR) diff --git a/mining/cpuminer/log.go b/mining/cpuminer/log.go index 7da6322c2..34e7132d1 100644 --- a/mining/cpuminer/log.go +++ b/mining/cpuminer/log.go @@ -10,4 +10,4 @@ import ( ) var log, _ = logger.Get(logger.SubsystemTags.MINR) -var spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) +var spawn = panics.GoroutineWrapperFunc(log) diff --git a/mining/simulator/log.go b/mining/simulator/log.go index e9144486c..7889e862b 100644 --- a/mining/simulator/log.go +++ b/mining/simulator/log.go @@ -11,7 +11,7 @@ import ( var ( backendLog = logs.NewBackend() log = backendLog.Logger("MNSM") - spawn = panics.GoroutineWrapperFunc(log, backendLog) + spawn = panics.GoroutineWrapperFunc(log) ) func initLog(logFile, errLogFile string) { @@ -30,5 +30,5 @@ func initLog(logFile, errLogFile string) { func enableRPCLogging() { rpclog := backendLog.Logger("RPCC") rpclog.SetLevel(logs.LevelTrace) - rpcclient.UseLogger(rpclog, backendLog) + rpcclient.UseLogger(rpclog) } diff --git a/mining/simulator/main.go b/mining/simulator/main.go index 272701d56..61cdd0448 100644 --- a/mining/simulator/main.go +++ b/mining/simulator/main.go @@ -11,7 +11,7 @@ import ( ) func main() { - defer panics.HandlePanic(log, backendLog, nil) + defer panics.HandlePanic(log, nil, nil) cfg, err := parseConfig() if err != nil { fmt.Fprintf(os.Stderr, "Error parsing command-line arguments: %s", err) diff --git a/netsync/log.go b/netsync/log.go index e66b7dae9..04b2d93dc 100644 --- a/netsync/log.go +++ b/netsync/log.go @@ -10,4 +10,4 @@ import ( ) var log, _ = logger.Get(logger.SubsystemTags.SYNC) -var spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) +var spawn = panics.GoroutineWrapperFuncWithPanicHandler(log) diff --git a/netsync/manager.go b/netsync/manager.go index fdd759049..b4c9c5d89 100644 --- a/netsync/manager.go +++ b/netsync/manager.go @@ -622,7 +622,7 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { if delay != 0 { spawn(func() { sm.QueueBlock(bmsg.block, bmsg.peer, true, make(chan struct{})) - }) + }, sm.handlePanic) } // Request the parents for the orphan block from the peer that sent it. @@ -1289,7 +1289,7 @@ func (sm *SyncManager) handleBlockDAGNotification(notification *blockdag.Notific if err != nil { panic(fmt.Sprintf("HandleNewBlock failed to handle block %s", block.Hash())) } - }) + }, sm.handlePanic) // Relay if we are current and the block was not just now unorphaned. // Otherwise peers that are current should already know about it @@ -1393,7 +1393,11 @@ func (sm *SyncManager) Start() { log.Trace("Starting sync manager") sm.wg.Add(1) - spawn(sm.blockHandler) + spawn(sm.blockHandler, sm.handlePanic) +} + +func (sm *SyncManager) handlePanic() { + atomic.AddInt32(&sm.shutdown, 1) } // Stop gracefully shuts down the sync manager by stopping all asynchronous diff --git a/peer/log.go b/peer/log.go index 9d3e0f4fc..76b8daf12 100644 --- a/peer/log.go +++ b/peer/log.go @@ -22,7 +22,7 @@ const ( ) var log, _ = logger.Get(logger.SubsystemTags.PEER) -var spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) +var spawn = panics.GoroutineWrapperFunc(log) // LogClosure is a closure that can be printed with %s to be used to // generate expensive-to-create data for a detailed log level and avoid doing diff --git a/rpcclient/log.go b/rpcclient/log.go index f26e01a8e..11b81e483 100644 --- a/rpcclient/log.go +++ b/rpcclient/log.go @@ -24,13 +24,13 @@ func init() { // by default until UseLogger is called. func DisableLog() { log = logs.Disabled - spawn = panics.GoroutineWrapperFunc(log, nil) + spawn = panics.GoroutineWrapperFunc(log) } // UseLogger uses a specified Logger to output package logging info. -func UseLogger(logger logs.Logger, backendLog *logs.Backend) { +func UseLogger(logger logs.Logger) { log = logger - spawn = panics.GoroutineWrapperFunc(log, backendLog) + spawn = panics.GoroutineWrapperFunc(log) } // LogClosure is a closure that can be printed with %s to be used to diff --git a/server/log.go b/server/log.go index 663a36da5..c70559fa9 100644 --- a/server/log.go +++ b/server/log.go @@ -11,5 +11,5 @@ import ( var ( log, _ = logger.Get(logger.SubsystemTags.SRVR) - spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(log) ) diff --git a/server/p2p/log.go b/server/p2p/log.go index 5b29e9e4d..bfdc83c07 100644 --- a/server/p2p/log.go +++ b/server/p2p/log.go @@ -12,7 +12,7 @@ import ( var ( srvrLog, _ = logger.Get(logger.SubsystemTags.SRVR) peerLog, _ = logger.Get(logger.SubsystemTags.PEER) - spawn = panics.GoroutineWrapperFunc(peerLog, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(peerLog) txmpLog, _ = logger.Get(logger.SubsystemTags.TXMP) indxLog, _ = logger.Get(logger.SubsystemTags.INDX) diff --git a/server/rpc/log.go b/server/rpc/log.go index 1bc633cec..5bf4942ae 100644 --- a/server/rpc/log.go +++ b/server/rpc/log.go @@ -11,5 +11,5 @@ import ( var ( log, _ = logger.Get(logger.SubsystemTags.RPCS) - spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(log) ) diff --git a/signal/signal.go b/signal/signal.go index a132551c9..ede7df38a 100644 --- a/signal/signal.go +++ b/signal/signal.go @@ -13,6 +13,10 @@ import ( // subsystems using the same code paths as when an interrupt signal is received. var ShutdownRequestChannel = make(chan struct{}) +// PanicShutdownChannel is used to initiate shutdown when any thread +// panics using the same code paths as when an interrupt signal is received. +var PanicShutdownChannel = make(chan struct{}) + // interruptSignals defines the default signals to catch in order to do a proper // shutdown. This may be modified during init depending on the platform. var interruptSignals = []os.Signal{os.Interrupt} @@ -30,11 +34,14 @@ func InterruptListener() <-chan struct{} { // channel to notify the caller. select { case sig := <-interruptChannel: - btcdLog.Infof("Received signal (%s). Shutting down...", + btcdLog.Infof("Received signal (%s). Shutting down...", sig) case <-ShutdownRequestChannel: - btcdLog.Info("Shutdown requested. Shutting down...") + btcdLog.Info("Shutdown requested. Shutting down...") + + case <-PanicShutdownChannel: + btcdLog.Info("Panic occurred. Shutting down...") } close(c) @@ -44,12 +51,17 @@ func InterruptListener() <-chan struct{} { for { select { case sig := <-interruptChannel: - btcdLog.Infof("Received signal (%s). Already "+ + btcdLog.Infof("Received signal (%s). Already "+ "shutting down...", sig) case <-ShutdownRequestChannel: - btcdLog.Info("Shutdown requested. Already " + + btcdLog.Info("Shutdown requested. Already " + "shutting down...") + + case <-PanicShutdownChannel: + btcdLog.Info("Panic occurred while shutting down. " + + "Forcing shut down...") + os.Exit(1) } } }() diff --git a/util/locks/log.go b/util/locks/log.go index d86d6a352..dac2814a3 100644 --- a/util/locks/log.go +++ b/util/locks/log.go @@ -11,5 +11,5 @@ import ( var ( log, _ = logger.Get(logger.SubsystemTags.UTIL) - spawn = panics.GoroutineWrapperFunc(log, logger.BackendLog) + spawn = panics.GoroutineWrapperFunc(log) ) diff --git a/util/panics/panics.go b/util/panics/panics.go index 812d71345..139bf9d00 100644 --- a/util/panics/panics.go +++ b/util/panics/panics.go @@ -2,31 +2,44 @@ package panics import ( "github.com/daglabs/btcd/logs" - "os" + "github.com/daglabs/btcd/signal" "runtime/debug" ) -// HandlePanic recovers panics, log them, and then exits the process. -func HandlePanic(log logs.Logger, backendLog *logs.Backend, goroutineStackTrace []byte) { +// HandlePanic recovers panics, log them, runs an optional panicHandler, +// and then initiates a clean shutdown. +func HandlePanic(log logs.Logger, goroutineStackTrace []byte, panicHandler func()) { if err := recover(); err != nil { log.Criticalf("Fatal error: %+v", err) if goroutineStackTrace != nil { - log.Criticalf("goroutine stack trance: %s", goroutineStackTrace) + log.Criticalf("goroutine stack trace: %s", goroutineStackTrace) } log.Criticalf("Stack trace: %s", debug.Stack()) - if backendLog != nil { - backendLog.Close() + if panicHandler != nil { + panicHandler() } - os.Exit(1) + signal.PanicShutdownChannel <- struct{}{} } } // GoroutineWrapperFunc returns a goroutine wrapper function that handles panics and write them to the log. -func GoroutineWrapperFunc(log logs.Logger, backendLog *logs.Backend) func(func()) { +func GoroutineWrapperFunc(log logs.Logger) func(func()) { return func(f func()) { stackTrace := debug.Stack() go func() { - defer HandlePanic(log, backendLog, stackTrace) + defer HandlePanic(log, stackTrace, nil) + f() + }() + } +} + +// GoroutineWrapperFuncWithPanicHandler returns a goroutine wrapper function that handles panics, +// write them to the log, and executes a handler function for panics. +func GoroutineWrapperFuncWithPanicHandler(log logs.Logger) func(func(), func()) { + return func(f func(), panicHandler func()) { + stackTrace := debug.Stack() + go func() { + defer HandlePanic(log, stackTrace, panicHandler) f() }() }