From 7c1495ba65bbd7eb624e2e03521602635fc96aeb Mon Sep 17 00:00:00 2001 From: stasatdaglabs <39559713+stasatdaglabs@users.noreply.github.com> Date: Thu, 28 Jan 2021 19:43:04 +0200 Subject: [PATCH 1/2] Force stop gRPC servers after a short timeout (#1463) * Force stop gRPC servers after a short timeout. * Use spawn instead of go. --- .../netadapter/server/grpcserver/grpc_server.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/infrastructure/network/netadapter/server/grpcserver/grpc_server.go b/infrastructure/network/netadapter/server/grpcserver/grpc_server.go index 3a5dd8737..9a303fe24 100644 --- a/infrastructure/network/netadapter/server/grpcserver/grpc_server.go +++ b/infrastructure/network/netadapter/server/grpcserver/grpc_server.go @@ -9,6 +9,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/peer" "net" + "time" ) type gRPCServer struct { @@ -61,7 +62,20 @@ func (s *gRPCServer) listenOn(listenAddr string) error { } func (s *gRPCServer) Stop() error { - s.server.GracefulStop() + const stopTimeout = 2 * time.Second + + stopChan := make(chan interface{}) + spawn("gRPCServer.Stop", func() { + s.server.GracefulStop() + close(stopChan) + }) + + select { + case <-stopChan: + case <-time.After(stopTimeout): + log.Warnf("Could not gracefully stop %s: timed out after %s", s.name, stopTimeout) + s.server.Stop() + } return nil } From 65e149b2bbae52d8ce9480dad45d91d82398b1ca Mon Sep 17 00:00:00 2001 From: stasatdaglabs <39559713+stasatdaglabs@users.noreply.github.com> Date: Fri, 29 Jan 2021 09:10:21 +0200 Subject: [PATCH 2/2] In kaspaminer, don't crash on submitBlock timeout (#1462) * In kaspaminer, don't crash on submitBlock timeout. * Make timeout messages have a log level of Warn. * Wait for a second after receiving a reject for IBD. Co-authored-by: Elichai Turkel --- cmd/kaspaminer/mineloop.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/kaspaminer/mineloop.go b/cmd/kaspaminer/mineloop.go index 718724d15..594f9811e 100644 --- a/cmd/kaspaminer/mineloop.go +++ b/cmd/kaspaminer/mineloop.go @@ -117,8 +117,14 @@ func handleFoundBlock(client *minerClient, block *externalapi.DomainBlock) error rejectReason, err := client.SubmitBlock(block) if err != nil { + if nativeerrors.Is(err, router.ErrTimeout) { + log.Warnf("Got timeout while submitting block %s to %s: %s", blockHash, client.Address(), err) + return nil + } if rejectReason == appmessage.RejectReasonIsInIBD { - log.Warnf("Block %s was rejected because the node is in IBD", blockHash) + const waitTime = 1 * time.Second + log.Warnf("Block %s was rejected because the node is in IBD. Waiting for %s", blockHash, waitTime) + time.Sleep(waitTime) return nil } return errors.Errorf("Error submitting block %s to %s: %s", blockHash, client.Address(), err) @@ -152,7 +158,7 @@ func templatesLoop(client *minerClient, miningAddr util.Address, getBlockTemplate := func() { template, err := client.GetBlockTemplate(miningAddr.String()) if nativeerrors.Is(err, router.ErrTimeout) { - log.Infof("Got timeout while requesting block template from %s", client.Address()) + log.Warnf("Got timeout while requesting block template from %s: %s", client.Address(), err) return } else if err != nil { errChan <- errors.Errorf("Error getting block template from %s: %s", client.Address(), err)