From 7b7fbbf8b8587bc0f74a6f69f67d2baea85a97db Mon Sep 17 00:00:00 2001
From: Hisanobu Tomari <posco.grubb@gmail.com>
Date: Sat, 8 Oct 2022 06:57:30 +0900
Subject: [PATCH] client/v3: Add backoff before retry when watch stream returns
 unavailable

The client retries connection without backoff when the server is gone
after the watch stream is established. This results in high CPU usage
in the client process. This change introduces backoff when the stream is
failed and unavailable.

Signed-off-by: Hisanobu Tomari <posco.grubb@gmail.com>
---
 clientv3/watch.go | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/clientv3/watch.go b/clientv3/watch.go
index c011177ba..0aaefdc2f 100644
--- a/clientv3/watch.go
+++ b/clientv3/watch.go
@@ -534,6 +534,7 @@ func (w *watchGrpcStream) run() {
 	cancelSet := make(map[int64]struct{})
 
 	var cur *pb.WatchResponse
+	backoff := time.Millisecond
 	for {
 		select {
 		// Watch() requested
@@ -678,6 +679,7 @@ func (w *watchGrpcStream) run() {
 				closeErr = err
 				return
 			}
+			backoff = w.backoffIfUnavailable(backoff, err)
 			if wc, closeErr = w.newWatchClient(); closeErr != nil {
 				return
 			}
@@ -1003,6 +1005,21 @@ func (w *watchGrpcStream) joinSubstreams() {
 
 var maxBackoff = 100 * time.Millisecond
 
+func (w *watchGrpcStream) backoffIfUnavailable(backoff time.Duration, err error) time.Duration {
+	if isUnavailableErr(w.ctx, err) {
+		// retry, but backoff
+		if backoff < maxBackoff {
+			// 25% backoff factor
+			backoff = backoff + backoff/4
+			if backoff > maxBackoff {
+				backoff = maxBackoff
+			}
+		}
+		time.Sleep(backoff)
+	}
+	return backoff
+}
+
 // openWatchClient retries opening a watch client until success or halt.
 // manually retry in case "ws==nil && err==nil"
 // TODO: remove FailFast=false
@@ -1023,17 +1040,7 @@ func (w *watchGrpcStream) openWatchClient() (ws pb.Watch_WatchClient, err error)
 		if isHaltErr(w.ctx, err) {
 			return nil, v3rpc.Error(err)
 		}
-		if isUnavailableErr(w.ctx, err) {
-			// retry, but backoff
-			if backoff < maxBackoff {
-				// 25% backoff factor
-				backoff = backoff + backoff/4
-				if backoff > maxBackoff {
-					backoff = maxBackoff
-				}
-			}
-			time.Sleep(backoff)
-		}
+		backoff = w.backoffIfUnavailable(backoff, err)
 	}
 	return ws, nil
 }