embed: add "InitialElectionTickAdvance"

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

embed/config.go

@@ -121,8 +121,38 @@ type Config struct {
 	// TickMs is the number of milliseconds between heartbeat ticks.
 	// TODO: decouple tickMs and heartbeat tick (current heartbeat tick = 1).
 	// make ticks a cluster wide configuration.
-	TickMs            uint  `json:"heartbeat-interval"`
+	TickMs     uint `json:"heartbeat-interval"`
-	ElectionMs        uint  `json:"election-timeout"`
+	ElectionMs uint `json:"election-timeout"`
+
+	// InitialElectionTickAdvance is true, then local member fast-forwards
+	// election ticks to speed up "initial" leader election trigger. This
+	// benefits the case of larger election ticks. For instance, cross
+	// datacenter deployment may require longer election timeout of 10-second.
+	// If true, local node does not need wait up to 10-second. Instead,
+	// forwards its election ticks to 8-second, and have only 2-second left
+	// before leader election.
+	//
+	// Major assumptions are that:
+	//  - cluster has no active leader thus advancing ticks enables faster
+	//    leader election, or
+	//  - cluster already has an established leader, and rejoining follower
+	//    is likely to receive heartbeats from the leader after tick advance
+	//    and before election timeout.
+	//
+	// However, when network from leader to rejoining follower is congested,
+	// and the follower does not receive leader heartbeat within left election
+	// ticks, disruptive election has to happen thus affecting cluster
+	// availabilities.
+	//
+	// Disabling this would slow down initial bootstrap process for cross
+	// datacenter deployments. Make your own tradeoffs by configuring
+	// --initial-election-tick-advance at the cost of slow initial bootstrap.
+	//
+	// If single-node, it advances ticks regardless.
+	//
+	// See https://github.com/coreos/etcd/issues/9333 for more detail.
+	InitialElectionTickAdvance bool `json:"initial-election-tick-advance"`
+
 	QuotaBackendBytes int64 `json:"quota-backend-bytes"`
 	MaxTxnOps         uint  `json:"max-txn-ops"`
 	MaxRequestBytes   uint  `json:"max-request-bytes"`
@@ -305,8 +335,9 @@ func NewConfig() *Config {
 		GRPCKeepAliveInterval: DefaultGRPCKeepAliveInterval,
 		GRPCKeepAliveTimeout:  DefaultGRPCKeepAliveTimeout,
 
-		TickMs:     100,
+		TickMs:                     100,
-		ElectionMs: 1000,
+		ElectionMs:                 1000,
+		InitialElectionTickAdvance: true,
 
 		LPUrls: []url.URL{*lpurl},
 		LCUrls: []url.URL{*lcurl},

embed/etcd.go

@@ -158,39 +158,40 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 	}
 
 	srvcfg := etcdserver.ServerConfig{
-		Name:                    cfg.Name,
+		Name:                       cfg.Name,
-		ClientURLs:              cfg.ACUrls,
+		ClientURLs:                 cfg.ACUrls,
-		PeerURLs:                cfg.APUrls,
+		PeerURLs:                   cfg.APUrls,
-		DataDir:                 cfg.Dir,
+		DataDir:                    cfg.Dir,
-		DedicatedWALDir:         cfg.WalDir,
+		DedicatedWALDir:            cfg.WalDir,
-		SnapCount:               cfg.SnapCount,
+		SnapCount:                  cfg.SnapCount,
-		MaxSnapFiles:            cfg.MaxSnapFiles,
+		MaxSnapFiles:               cfg.MaxSnapFiles,
-		MaxWALFiles:             cfg.MaxWalFiles,
+		MaxWALFiles:                cfg.MaxWalFiles,
-		InitialPeerURLsMap:      urlsmap,
+		InitialPeerURLsMap:         urlsmap,
-		InitialClusterToken:     token,
+		InitialClusterToken:        token,
-		DiscoveryURL:            cfg.Durl,
+		DiscoveryURL:               cfg.Durl,
-		DiscoveryProxy:          cfg.Dproxy,
+		DiscoveryProxy:             cfg.Dproxy,
-		NewCluster:              cfg.IsNewCluster(),
+		NewCluster:                 cfg.IsNewCluster(),
-		PeerTLSInfo:             cfg.PeerTLSInfo,
+		PeerTLSInfo:                cfg.PeerTLSInfo,
-		TickMs:                  cfg.TickMs,
+		TickMs:                     cfg.TickMs,
-		ElectionTicks:           cfg.ElectionTicks(),
+		ElectionTicks:              cfg.ElectionTicks(),
-		AutoCompactionRetention: autoCompactionRetention,
+		InitialElectionTickAdvance: cfg.InitialElectionTickAdvance,
-		AutoCompactionMode:      cfg.AutoCompactionMode,
+		AutoCompactionRetention:    autoCompactionRetention,
-		QuotaBackendBytes:       cfg.QuotaBackendBytes,
+		AutoCompactionMode:         cfg.AutoCompactionMode,
-		MaxTxnOps:               cfg.MaxTxnOps,
+		QuotaBackendBytes:          cfg.QuotaBackendBytes,
-		MaxRequestBytes:         cfg.MaxRequestBytes,
+		MaxTxnOps:                  cfg.MaxTxnOps,
-		StrictReconfigCheck:     cfg.StrictReconfigCheck,
+		MaxRequestBytes:            cfg.MaxRequestBytes,
-		ClientCertAuthEnabled:   cfg.ClientTLSInfo.ClientCertAuth,
+		StrictReconfigCheck:        cfg.StrictReconfigCheck,
-		AuthToken:               cfg.AuthToken,
+		ClientCertAuthEnabled:      cfg.ClientTLSInfo.ClientCertAuth,
-		CORS:                    cfg.CORS,
+		AuthToken:                  cfg.AuthToken,
-		HostWhitelist:           cfg.HostWhitelist,
+		CORS:                       cfg.CORS,
-		InitialCorruptCheck:     cfg.ExperimentalInitialCorruptCheck,
+		HostWhitelist:              cfg.HostWhitelist,
-		CorruptCheckTime:        cfg.ExperimentalCorruptCheckTime,
+		InitialCorruptCheck:        cfg.ExperimentalInitialCorruptCheck,
-		PreVote:                 cfg.PreVote,
+		CorruptCheckTime:           cfg.ExperimentalCorruptCheckTime,
-		Logger:                  cfg.logger,
+		PreVote:                    cfg.PreVote,
-		LoggerConfig:            cfg.loggerConfig,
+		Logger:                     cfg.logger,
-		Debug:                   cfg.Debug,
+		LoggerConfig:               cfg.loggerConfig,
-		ForceNewCluster:         cfg.ForceNewCluster,
+		Debug:                      cfg.Debug,
+		ForceNewCluster:            cfg.ForceNewCluster,
 	}
 	if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
 		return e, err