[NOD-899] Inside the database, in case we're out of disk space, panic without printing the stack trace (#691)

* [NOD-899] Inside the database, in case we're out of disk space, panic without printing the stack trace.

* [NOD-899] Fix bad variable name.

* [NOD-899] Reduce code duplication.
This commit is contained in:
stasatdaglabs 2020-04-06 16:00:48 +03:00 committed by GitHub
parent a31139d4a5
commit 3c4a80f16d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 36 additions and 22 deletions

View File

@ -1,6 +1,7 @@
package ff
import (
"github.com/kaspanet/kaspad/util/panics"
"github.com/pkg/errors"
"hash/crc32"
"os"
@ -154,7 +155,7 @@ func (s *flatFileStore) writeData(data []byte, fieldName string) error {
if err != nil {
var pathErr *os.PathError
if ok := errors.As(err, &pathErr); ok && pathErr.Err == syscall.ENOSPC {
panic("No space left on the hard disk, exiting...")
panics.Exit(log, "No space left on the hard disk.")
}
return errors.Wrapf(err, "failed to write %s in store %s to file %d "+
"at offset %d", fieldName, s.storeName, cursor.currentFileNumber,

View File

@ -9,33 +9,17 @@ import (
"github.com/kaspanet/kaspad/logs"
)
// HandlePanic recovers panics, log them, runs an optional panicHandler,
// and then initiates a clean shutdown.
const exitHandlerTimeout = 5 * time.Second
// HandlePanic recovers panics and then initiates a clean shutdown.
func HandlePanic(log *logs.Logger, goroutineStackTrace []byte) {
err := recover()
if err == nil {
return
}
panicHandlerDone := make(chan struct{})
go func() {
log.Criticalf("Fatal error: %+v", err)
if goroutineStackTrace != nil {
log.Criticalf("Goroutine stack trace: %s", goroutineStackTrace)
}
log.Criticalf("Stack trace: %s", debug.Stack())
log.Backend().Close()
close(panicHandlerDone)
}()
const panicHandlerTimeout = 5 * time.Second
select {
case <-time.After(panicHandlerTimeout):
fmt.Fprintln(os.Stderr, "Couldn't handle a fatal error. Exiting...")
case <-panicHandlerDone:
}
log.Criticalf("Exiting")
os.Exit(1)
reason := fmt.Sprintf("Fatal error: %+v", err)
exit(log, reason, debug.Stack(), goroutineStackTrace)
}
// GoroutineWrapperFunc returns a goroutine wrapper function that handles panics and writes them to the log.
@ -59,3 +43,32 @@ func AfterFuncWrapperFunc(log *logs.Logger) func(d time.Duration, f func()) *tim
})
}
}
// Exit prints the given reason to log and initiates a clean shutdown.
func Exit(log *logs.Logger, reason string) {
exit(log, reason, nil, nil)
}
// Exit prints the given reason, prints either of the given stack traces (if not nil),
// waits for them to finish writing, and exits.
func exit(log *logs.Logger, reason string, currentThreadStackTrace []byte, goroutineStackTrace []byte) {
exitHandlerDone := make(chan struct{})
go func() {
log.Criticalf("Exiting: %s", reason)
if goroutineStackTrace != nil {
log.Criticalf("Goroutine stack trace: %s", goroutineStackTrace)
}
if currentThreadStackTrace != nil {
log.Criticalf("Stack trace: %s", currentThreadStackTrace)
}
log.Backend().Close()
close(exitHandlerDone)
}()
select {
case <-time.After(exitHandlerTimeout):
fmt.Fprintln(os.Stderr, "Couldn't exit gracefully.")
case <-exitHandlerDone:
}
os.Exit(1)
}