functional-tester: defrag every 500 round

Fix https://github.com/coreos/etcd/issues/4665.

tools/functional-tester/etcd-tester/cluster.go

@@ -376,3 +376,20 @@ func (c *cluster) checkCompact(rev int64) error {
 	}
 	return nil
 }
+
+func (c *cluster) defrag() error {
+	for _, u := range c.GRPCURLs {
+		plog.Printf("defragmenting %s\n", u)
+		conn, err := grpc.Dial(u, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second))
+		if err != nil {
+			return err
+		}
+		mt := pb.NewMaintenanceClient(conn)
+		if _, err = mt.Defragment(context.Background(), &pb.DefragmentRequest{}); err != nil {
+			return err
+		}
+		conn.Close()
+		plog.Printf("defragmented %s\n", u)
+	}
+	return nil
+}

tools/functional-tester/etcd-tester/tester.go

@@ -158,6 +158,29 @@ func (tt *tester) runLoop() {
 			}
 		}
 		plog.Printf("[round#%d] confirmed compaction at %d", i, revToCompact)
+
+		if i > 0 && i%500 == 0 { // every 500 rounds
+			plog.Printf("[round#%d] canceling the stressers...", i)
+			for _, s := range tt.cluster.Stressers {
+				s.Cancel()
+			}
+			plog.Printf("[round#%d] canceled stressers", i)
+
+			plog.Printf("[round#%d] deframenting...", i)
+			if err := tt.cluster.defrag(); err != nil {
+				plog.Printf("[round#%d] defrag error (%v)", i, err)
+				if err := tt.cleanup(i, 0); err != nil {
+					plog.Printf("[round#%d] cleanup error: %v", i, err)
+					return
+				}
+			}
+			plog.Printf("[round#%d] deframented...", i)
+
+			plog.Printf("[round#%d] restarting the stressers...", i)
+			for _, s := range tt.cluster.Stressers {
+				go s.Stress()
+			}
+		}
 	}
 }