From df2cc4bc8ca83b76daa91462847081280f65d387 Mon Sep 17 00:00:00 2001 From: Iwasaki Yudai Date: Fri, 16 Jun 2017 14:53:25 -0700 Subject: [PATCH] grpcproxy: Disable fast fail on lease grant call to cluster Problem Observed ---------------- When there is no etcd process behind the proxy, clients repeat resending lease grant requests without delay. This behavior can cause abnormal resource consumption on CPU/RAM and network. Problem Detail -------------- `LeaseGrant()` uses a bare protobuf client to forward requests. However, it doesn't use `grpc.FailFast(false)`, which means the method returns an `Unavailable` error immediately when no etcd process is available. In clientv3, `Unavailable` errors are not considered the "Halt" error, and library retries the request without delay. Both clients and the proxy consume much CPU cycles to process retry requests. Resolution ---------- Add `grpc.FailFast(false))` to `LeaseGrant()` of the `leaseProxy`. This makes the proxy not to return immediately when no etcd process is available. Clients will simply timeout requests instead. --- proxy/grpcproxy/lease.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/grpcproxy/lease.go b/proxy/grpcproxy/lease.go index 19c2249a7..cd7481da8 100644 --- a/proxy/grpcproxy/lease.go +++ b/proxy/grpcproxy/lease.go @@ -73,7 +73,7 @@ func NewLeaseProxy(c *clientv3.Client) (pb.LeaseServer, <-chan struct{}) { } func (lp *leaseProxy) LeaseGrant(ctx context.Context, cr *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { - rp, err := lp.leaseClient.LeaseGrant(ctx, cr) + rp, err := lp.leaseClient.LeaseGrant(ctx, cr, grpc.FailFast(false)) if err != nil { return nil, err }