diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index ef7dcc283..6cc3e08e5 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -28,6 +28,7 @@ import ( "go.etcd.io/etcd/pkg/v3/flags" "go.etcd.io/etcd/pkg/v3/logutil" "go.etcd.io/etcd/server/v3/embed" + "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.uber.org/zap" "sigs.k8s.io/yaml" @@ -161,6 +162,10 @@ func newConfig() *config { fs.DurationVar(&cfg.ec.GRPCKeepAliveInterval, "grpc-keepalive-interval", cfg.ec.GRPCKeepAliveInterval, "Frequency duration of server-to-client ping to check if a connection is alive (0 to disable).") fs.DurationVar(&cfg.ec.GRPCKeepAliveTimeout, "grpc-keepalive-timeout", cfg.ec.GRPCKeepAliveTimeout, "Additional duration of wait before closing a non-responsive connection (0 to disable).") + // raft connection timeouts + fs.DurationVar(&rafthttp.ConnReadTimeout, "raft-read-timeout", rafthttp.DefaultConnReadTimeout, "Read timeout set on each rafthttp connection") + fs.DurationVar(&rafthttp.ConnWriteTimeout, "raft-write-timeout", rafthttp.DefaultConnWriteTimeout, "Write timeout set on each rafthttp connection") + // clustering fs.Var( flags.NewUniqueURLsWithExceptions(embed.DefaultInitialAdvertisePeerURLs, ""), @@ -337,6 +342,15 @@ func (cfg *config) configFromCmdLine() error { return err } + if rafthttp.ConnReadTimeout < rafthttp.DefaultConnReadTimeout { + rafthttp.ConnReadTimeout = rafthttp.DefaultConnReadTimeout + lg.Info(fmt.Sprintf("raft-read-timeout increased to minimum value: %v", rafthttp.DefaultConnReadTimeout)) + } + if rafthttp.ConnWriteTimeout < rafthttp.DefaultConnWriteTimeout { + rafthttp.ConnWriteTimeout = rafthttp.DefaultConnWriteTimeout + lg.Info(fmt.Sprintf("raft-write-timeout increased to minimum value: %v", rafthttp.DefaultConnWriteTimeout)) + } + cfg.ec.LPUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-peer-urls") cfg.ec.APUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "initial-advertise-peer-urls") cfg.ec.LCUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-client-urls") diff --git a/server/etcdserver/api/rafthttp/peer.go b/server/etcdserver/api/rafthttp/peer.go index 136bc075c..ab6881eee 100644 --- a/server/etcdserver/api/rafthttp/peer.go +++ b/server/etcdserver/api/rafthttp/peer.go @@ -37,8 +37,8 @@ const ( // to keep the connection alive. // For short term pipeline connections, the connection MUST be killed to avoid it being // put back to http pkg connection pool. - ConnReadTimeout = 5 * time.Second - ConnWriteTimeout = 5 * time.Second + DefaultConnReadTimeout = 5 * time.Second + DefaultConnWriteTimeout = 5 * time.Second recvBufSize = 4096 // maxPendingProposals holds the proposals during one leader election process. @@ -55,6 +55,11 @@ const ( sendSnap = "sendMsgSnap" ) +var ( + ConnReadTimeout = DefaultConnReadTimeout + ConnWriteTimeout = DefaultConnWriteTimeout +) + type Peer interface { // send sends the message to the remote peer. The function is non-blocking // and has no promise that the message will be received by the remote. diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index 5c5a9fb67..01fdc0c3f 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -85,10 +85,6 @@ const ( HealthInterval = 5 * time.Second purgeFileInterval = 30 * time.Second - // monitorVersionInterval should be smaller than the timeout - // on the connection. Or we will not be able to reuse the connection - // (since it will timeout). - monitorVersionInterval = rafthttp.ConnWriteTimeout - time.Second // max number of in-flight snapshot messages etcdserver allows to have // This number is more than enough for most clusters with 5 machines. @@ -107,6 +103,11 @@ const ( ) var ( + // monitorVersionInterval should be smaller than the timeout + // on the connection. Or we will not be able to reuse the connection + // (since it will timeout). + monitorVersionInterval = rafthttp.ConnWriteTimeout - time.Second + recommendedMaxRequestBytesString = humanize.Bytes(uint64(recommendedMaxRequestBytes)) storeMemberAttributeRegexp = regexp.MustCompile(path.Join(membership.StoreMembersPrefix, "[[:xdigit:]]{1,16}", "attributes")) )