From 0926a434b72b151f717b4f59c00ceb6b159e6a74 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 7 Aug 2019 16:03:01 -0700 Subject: [PATCH 1/3] functional.yaml: try lower snapshot count for flaky tests, error threshold Signed-off-by: Gyuho Lee --- functional.yaml | 6 +++--- functional/tester/cluster_run.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/functional.yaml b/functional.yaml index 9b5221cc0..7ae03ca79 100644 --- a/functional.yaml +++ b/functional.yaml @@ -29,7 +29,7 @@ agent-configs: initial-cluster: s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381 initial-cluster-state: new initial-cluster-token: tkn - snapshot-count: 10000 + snapshot-count: 2000 quota-backend-bytes: 10740000000 # 10 GiB pre-vote: true initial-corrupt-check: true @@ -80,7 +80,7 @@ agent-configs: initial-cluster: s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381 initial-cluster-state: new initial-cluster-token: tkn - snapshot-count: 10000 + snapshot-count: 2000 quota-backend-bytes: 10740000000 # 10 GiB pre-vote: true initial-corrupt-check: true @@ -131,7 +131,7 @@ agent-configs: initial-cluster: s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381 initial-cluster-state: new initial-cluster-token: tkn - snapshot-count: 10000 + snapshot-count: 2000 quota-backend-bytes: 10740000000 # 10 GiB pre-vote: true initial-corrupt-check: true diff --git a/functional/tester/cluster_run.go b/functional/tester/cluster_run.go index f4091e0d8..0578c1323 100644 --- a/functional/tester/cluster_run.go +++ b/functional/tester/cluster_run.go @@ -212,8 +212,8 @@ func (clus *Cluster) doRound() error { ) // with network delay, some ongoing requests may fail - // only return error, if more than 10% of QPS requests fail - if cnt > int(clus.Tester.StressQPS)/10 { + // only return error, if more than 30% of QPS requests fail + if cnt > int(float64(clus.Tester.StressQPS)*0.3) { return fmt.Errorf("expected no error in %q, got %q", fcase.String(), ess) } } From d1c7be24b07d0b9266ad656e0297fd085a9c52b6 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Wed, 7 Aug 2019 17:36:04 -0700 Subject: [PATCH 2/3] functional/rpcpb: make client log less verbose Signed-off-by: Gyuho Lee --- functional/rpcpb/member.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/functional/rpcpb/member.go b/functional/rpcpb/member.go index 5b400bdc9..930c371c7 100644 --- a/functional/rpcpb/member.go +++ b/functional/rpcpb/member.go @@ -25,6 +25,7 @@ import ( "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/clientv3/snapshot" pb "go.etcd.io/etcd/etcdserver/etcdserverpb" + "go.etcd.io/etcd/pkg/logutil" "go.etcd.io/etcd/pkg/transport" "github.com/dustin/go-humanize" @@ -94,10 +95,19 @@ func (m *Member) CreateEtcdClientConfig(opts ...grpc.DialOption) (cfg *clientv3. } } + // TODO: make this configurable + level := "error" + if os.Getenv("ETCD_CLIENT_DEBUG") != "" { + level = "debug" + } + lcfg := logutil.DefaultZapLoggerConfig + lcfg.Level = zap.NewAtomicLevelAt(logutil.ConvertToZapLevel(level)) + cfg = &clientv3.Config{ Endpoints: []string{m.EtcdClientEndpoint}, DialTimeout: 10 * time.Second, DialOptions: opts, + LogConfig: &lcfg, } if secure { // assume save TLS assets are already stord on disk From 72e00cea3aa0d749a13107788a588ade6a92d8bf Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 8 Aug 2019 08:12:05 -0700 Subject: [PATCH 3/3] functional/agent: copy file, instead of renaming To retain failure logs in CI testing. Signed-off-by: Gyuho Lee --- functional/agent/utils.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/functional/agent/utils.go b/functional/agent/utils.go index c0dd92fc5..e741d11fd 100644 --- a/functional/agent/utils.go +++ b/functional/agent/utils.go @@ -15,6 +15,7 @@ package agent import ( + "io" "net" "net/url" "os" @@ -36,7 +37,8 @@ func archive(baseDir, etcdLogPath, dataDir string) error { return err } - if err := os.Rename(etcdLogPath, filepath.Join(dir, "etcd.log")); err != nil { + dst := filepath.Join(dir, "etcd.log") + if err := copyFile(etcdLogPath, dst); err != nil { if !os.IsNotExist(err) { return err } @@ -79,6 +81,25 @@ func getURLAndPort(addr string) (urlAddr *url.URL, port int, err error) { return urlAddr, port, err } +func copyFile(src, dst string) error { + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + w, err := os.Create(dst) + if err != nil { + return err + } + defer w.Close() + + if _, err = io.Copy(w, f); err != nil { + return err + } + return w.Sync() +} + func cleanPageCache() error { // https://www.kernel.org/doc/Documentation/sysctl/vm.txt // https://github.com/torvalds/linux/blob/master/fs/drop_caches.c