From 4998db4e6408d212ae775f9dcced659237eedeb9 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 5 Apr 2018 15:34:03 -0700 Subject: [PATCH] functional-tester/tester: send TLS client requests, cleanup Signed-off-by: Gyuho Lee --- tools/functional-tester/tester/cluster.go | 87 +++++++++++++++++-- .../functional-tester/tester/cluster_test.go | 4 +- .../tester/cluster_tester.go | 12 ++- .../functional-tester/tester/local-test.yaml | 4 +- 4 files changed, 94 insertions(+), 13 deletions(-) diff --git a/tools/functional-tester/tester/cluster.go b/tools/functional-tester/tester/cluster.go index 5a1e178c0..68b54bce1 100644 --- a/tools/functional-tester/tester/cluster.go +++ b/tools/functional-tester/tester/cluster.go @@ -27,6 +27,7 @@ import ( "time" "github.com/coreos/etcd/pkg/debugutil" + "github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/tools/functional-tester/rpcpb" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -201,7 +202,8 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) { if mem.Etcd.ClientAutoTLS || mem.Etcd.ClientCertFile != "" { for _, cu := range mem.Etcd.ListenClientURLs { - u, err := url.Parse(cu) + var u *url.URL + u, err = url.Parse(cu) if err != nil { return nil, err } @@ -210,7 +212,8 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) { } } for _, cu := range mem.Etcd.AdvertiseClientURLs { - u, err := url.Parse(cu) + var u *url.URL + u, err = url.Parse(cu) if err != nil { return nil, err } @@ -221,7 +224,8 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) { } if mem.Etcd.PeerAutoTLS || mem.Etcd.PeerCertFile != "" { for _, cu := range mem.Etcd.ListenPeerURLs { - u, err := url.Parse(cu) + var u *url.URL + u, err = url.Parse(cu) if err != nil { return nil, err } @@ -230,7 +234,8 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) { } } for _, cu := range mem.Etcd.AdvertisePeerURLs { - u, err := url.Parse(cu) + var u *url.URL + u, err = url.Parse(cu) if err != nil { return nil, err } @@ -619,9 +624,79 @@ func (clus *Cluster) sendOperation(idx int, op rpcpb.Operation) error { } if !resp.Success { - err = errors.New(resp.Status) + return errors.New(resp.Status) } - return err + + m, secure := clus.Members[idx], false + for _, cu := range m.Etcd.AdvertiseClientURLs { + u, err := url.Parse(cu) + if err != nil { + return err + } + if u.Scheme == "https" { // TODO: handle unix + secure = true + } + } + + // store TLS assets from agents/servers onto disk + if secure && (op == rpcpb.Operation_InitialStartEtcd || op == rpcpb.Operation_RestartEtcd) { + dirClient := filepath.Join( + clus.Tester.TesterDataDir, + clus.Members[idx].Etcd.Name, + "fixtures", + "client", + ) + if err = fileutil.TouchDirAll(dirClient); err != nil { + return err + } + + clientCertData := []byte(resp.Member.ClientCertData) + if len(clientCertData) == 0 { + return fmt.Errorf("got empty client cert from %q", m.EtcdClientEndpoint) + } + clientCertPath := filepath.Join(dirClient, "cert.pem") + if err = ioutil.WriteFile(clientCertPath, clientCertData, 0644); err != nil { // overwrite if exists + return err + } + resp.Member.ClientCertPath = clientCertPath + clus.lg.Info( + "saved client cert file", + zap.String("path", clientCertPath), + ) + + clientKeyData := []byte(resp.Member.ClientKeyData) + if len(clientKeyData) == 0 { + return fmt.Errorf("got empty client key from %q", m.EtcdClientEndpoint) + } + clientKeyPath := filepath.Join(dirClient, "key.pem") + if err = ioutil.WriteFile(clientKeyPath, clientKeyData, 0644); err != nil { // overwrite if exists + return err + } + resp.Member.ClientKeyPath = clientKeyPath + clus.lg.Info( + "saved client key file", + zap.String("path", clientKeyPath), + ) + + clientTrustedCAData := []byte(resp.Member.ClientTrustedCAData) + if len(clientTrustedCAData) != 0 { + // TODO: disable this when auto TLS is deprecated + clientTrustedCAPath := filepath.Join(dirClient, "ca.pem") + if err = ioutil.WriteFile(clientTrustedCAPath, clientTrustedCAData, 0644); err != nil { // overwrite if exists + return err + } + resp.Member.ClientTrustedCAPath = clientTrustedCAPath + clus.lg.Info( + "saved client trusted CA file", + zap.String("path", clientTrustedCAPath), + ) + } + + // no need to store peer certs for tester clients + + clus.Members[idx] = resp.Member + } + return nil } // DestroyEtcdAgents terminates all tester connections to agents and etcd servers. diff --git a/tools/functional-tester/tester/cluster_test.go b/tools/functional-tester/tester/cluster_test.go index 1a5d88ab6..e2b34e9e4 100644 --- a/tools/functional-tester/tester/cluster_test.go +++ b/tools/functional-tester/tester/cluster_test.go @@ -33,7 +33,6 @@ func Test_newCluster(t *testing.T) { FailpointHTTPAddr: "http://127.0.0.1:7381", BaseDir: "/tmp/etcd-agent-data-1", EtcdLogPath: "/tmp/etcd-agent-data-1/current-etcd.log", - EtcdClientTLS: false, EtcdClientProxy: false, EtcdPeerProxy: true, EtcdClientEndpoint: "127.0.0.1:1379", @@ -72,7 +71,6 @@ func Test_newCluster(t *testing.T) { FailpointHTTPAddr: "http://127.0.0.1:7382", BaseDir: "/tmp/etcd-agent-data-2", EtcdLogPath: "/tmp/etcd-agent-data-2/current-etcd.log", - EtcdClientTLS: false, EtcdClientProxy: false, EtcdPeerProxy: true, EtcdClientEndpoint: "127.0.0.1:2379", @@ -111,7 +109,6 @@ func Test_newCluster(t *testing.T) { FailpointHTTPAddr: "http://127.0.0.1:7383", BaseDir: "/tmp/etcd-agent-data-3", EtcdLogPath: "/tmp/etcd-agent-data-3/current-etcd.log", - EtcdClientTLS: false, EtcdClientProxy: false, EtcdPeerProxy: true, EtcdClientEndpoint: "127.0.0.1:3379", @@ -146,6 +143,7 @@ func Test_newCluster(t *testing.T) { }, }, Tester: &rpcpb.Tester{ + TesterDataDir: "/tmp/etcd-tester-data", TesterNetwork: "tcp", TesterAddr: "127.0.0.1:9028", DelayLatencyMs: 5000, diff --git a/tools/functional-tester/tester/cluster_tester.go b/tools/functional-tester/tester/cluster_tester.go index 1db87a457..b30c568dc 100644 --- a/tools/functional-tester/tester/cluster_tester.go +++ b/tools/functional-tester/tester/cluster_tester.go @@ -19,6 +19,7 @@ import ( "os" "time" + "github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/tools/functional-tester/rpcpb" "go.uber.org/zap" @@ -30,7 +31,13 @@ const compactQPS = 50000 // StartTester starts tester. func (clus *Cluster) StartTester() { - // TODO: upate status + if err := fileutil.TouchDirAll(clus.Tester.TesterDataDir); err != nil { + clus.lg.Panic( + "failed to create test data directory", + zap.String("dir", clus.Tester.TesterDataDir), + zap.Error(err), + ) + } var preModifiedKey int64 for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ { @@ -124,6 +131,7 @@ func (clus *Cluster) doRound() error { zap.Int("round", clus.rd), zap.Int("case", clus.cs), zap.String("desc", fa.Desc()), + zap.Int("total-failures", len(clus.failures)), ) clus.lg.Info("wait health before injecting failures") @@ -208,6 +216,7 @@ func (clus *Cluster) doRound() error { zap.Int("round", clus.rd), zap.Int("case", clus.cs), zap.String("desc", fa.Desc()), + zap.Int("total-failures", len(clus.failures)), zap.Duration("took", time.Since(caseNow)), ) } @@ -216,6 +225,7 @@ func (clus *Cluster) doRound() error { "round ALL PASS", zap.Int("round", clus.rd), zap.Strings("failures", clus.failureStrings()), + zap.Int("total-failures", len(clus.failures)), zap.Duration("took", time.Since(roundNow)), ) return nil diff --git a/tools/functional-tester/tester/local-test.yaml b/tools/functional-tester/tester/local-test.yaml index 8cd09d84b..aab38036f 100644 --- a/tools/functional-tester/tester/local-test.yaml +++ b/tools/functional-tester/tester/local-test.yaml @@ -4,7 +4,6 @@ agent-configs: failpoint-http-addr: http://127.0.0.1:7381 base-dir: /tmp/etcd-agent-data-1 etcd-log-path: /tmp/etcd-agent-data-1/current-etcd.log - etcd-client-tls: false etcd-client-proxy: false etcd-peer-proxy: true etcd-client-endpoint: 127.0.0.1:1379 @@ -40,7 +39,6 @@ agent-configs: failpoint-http-addr: http://127.0.0.1:7382 base-dir: /tmp/etcd-agent-data-2 etcd-log-path: /tmp/etcd-agent-data-2/current-etcd.log - etcd-client-tls: false etcd-client-proxy: false etcd-peer-proxy: true etcd-client-endpoint: 127.0.0.1:2379 @@ -76,7 +74,6 @@ agent-configs: failpoint-http-addr: http://127.0.0.1:7383 base-dir: /tmp/etcd-agent-data-3 etcd-log-path: /tmp/etcd-agent-data-3/current-etcd.log - etcd-client-tls: false etcd-client-proxy: false etcd-peer-proxy: true etcd-client-endpoint: 127.0.0.1:3379 @@ -109,6 +106,7 @@ agent-configs: initial-corrupt-check: true tester-config: + tester-data-dir: /tmp/etcd-tester-data tester-network: tcp tester-addr: 127.0.0.1:9028