From eb614c35f75c4ea0bf21f11085c831a1a1ed53a2 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Tue, 14 Mar 2023 15:44:16 +0100 Subject: [PATCH] tests: Add connection muiltiplexer testing Signed-off-by: Marek Siarkowicz --- bill-of-materials.json | 9 ++ tests/e2e/cmux_test.go | 200 +++++++++++++++++++++++++++++++++++++++++ tests/go.mod | 2 +- 3 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/cmux_test.go diff --git a/bill-of-materials.json b/bill-of-materials.json index 7de0322ce..7c22d9daa 100644 --- a/bill-of-materials.json +++ b/bill-of-materials.json @@ -611,6 +611,15 @@ } ] }, + { + "project": "golang.org/x/sync/errgroup", + "licenses": [ + { + "type": "BSD 3-clause \"New\" or \"Revised\" License", + "confidence": 0.9663865546218487 + } + ] + }, { "project": "golang.org/x/sys/unix", "licenses": [ diff --git a/tests/e2e/cmux_test.go b/tests/e2e/cmux_test.go new file mode 100644 index 000000000..1d79bef79 --- /dev/null +++ b/tests/e2e/cmux_test.go @@ -0,0 +1,200 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// These tests are directly validating etcd connection multiplexing. +//go:build !cluster_proxy + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/prometheus/common/expfmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + pb "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/mvccpb" + "go.etcd.io/etcd/api/v3/version" + "go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp" +) + +func TestConnectionMultiplexing(t *testing.T) { + BeforeTest(t) + for _, tc := range []struct { + name string + serverTLS clientConnType + }{ + { + name: "ServerTLS", + serverTLS: clientTLS, + }, + { + name: "ServerNonTLS", + serverTLS: clientNonTLS, + }, + { + name: "ServerTLSAndNonTLS", + serverTLS: clientTLSAndNonTLS, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + cfg := etcdProcessClusterConfig{clusterSize: 1, clientTLS: tc.serverTLS} + clus, err := newEtcdProcessCluster(t, &cfg) + require.NoError(t, err) + defer clus.Close() + + var clientScenarios []clientConnType + switch tc.serverTLS { + case clientTLS: + clientScenarios = []clientConnType{clientTLS} + case clientNonTLS: + clientScenarios = []clientConnType{clientNonTLS} + case clientTLSAndNonTLS: + clientScenarios = []clientConnType{clientTLS, clientNonTLS} + } + + for _, connType := range clientScenarios { + name := "ClientNonTLS" + if connType == clientTLS { + name = "ClientTLS" + } + t.Run(name, func(t *testing.T) { + testConnectionMultiplexing(ctx, t, clus.EndpointsV3()[0], connType) + }) + } + }) + } + +} + +func testConnectionMultiplexing(ctx context.Context, t *testing.T, endpoint string, connType clientConnType) { + switch connType { + case clientTLS: + endpoint = toTLS(endpoint) + case clientNonTLS: + default: + panic(fmt.Sprintf("Unsupported conn type %v", connType)) + } + t.Run("etcdctl", func(t *testing.T) { + etcdctl := NewEtcdctl([]string{endpoint}, connType, false) + _, err := etcdctl.Get("a") + assert.NoError(t, err) + }) + t.Run("clientv3", func(t *testing.T) { + c := newClient(t, []string{endpoint}, connType, false) + _, err := c.Get(ctx, "a") + assert.NoError(t, err) + }) + t.Run("curl", func(t *testing.T) { + for _, httpVersion := range []string{"2", "1.1", "1.0", ""} { + tname := "http" + httpVersion + if httpVersion == "" { + tname = "default" + } + t.Run(tname, func(t *testing.T) { + assert.NoError(t, fetchGrpcGateway(endpoint, httpVersion, connType)) + assert.NoError(t, fetchMetrics(endpoint, httpVersion, connType)) + assert.NoError(t, fetchVersion(endpoint, httpVersion, connType)) + assert.NoError(t, fetchHealth(endpoint, httpVersion, connType)) + assert.NoError(t, fetchDebugVars(endpoint, httpVersion, connType)) + }) + } + }) +} + +func fetchGrpcGateway(endpoint string, httpVersion string, connType clientConnType) error { + rangeData, err := json.Marshal(&pb.RangeRequest{ + Key: []byte("a"), + }) + if err != nil { + return err + } + req := cURLReq{endpoint: "/v3/kv/range", value: string(rangeData), timeout: 5, httpVersion: httpVersion} + respData, err := curl(endpoint, "POST", req, connType) + return validateGrpcgatewayRangeReponse([]byte(respData)) +} + +func validateGrpcgatewayRangeReponse(respData []byte) error { + // Modified json annotation so ResponseHeader fields are stored in string. + type responseHeader struct { + ClusterId uint64 `json:"cluster_id,string,omitempty"` + MemberId uint64 `json:"member_id,string,omitempty"` + Revision int64 `json:"revision,string,omitempty"` + RaftTerm uint64 `json:"raft_term,string,omitempty"` + } + type rangeResponse struct { + Header *responseHeader `json:"header,omitempty"` + Kvs []*mvccpb.KeyValue `json:"kvs,omitempty"` + More bool `json:"more,omitempty"` + Count int64 `json:"count,omitempty"` + } + var resp rangeResponse + return json.Unmarshal(respData, &resp) +} + +func fetchMetrics(endpoint string, httpVersion string, connType clientConnType) error { + req := cURLReq{endpoint: "/metrics", timeout: 5, httpVersion: httpVersion} + respData, err := curl(endpoint, "GET", req, connType) + if err != nil { + return err + } + var parser expfmt.TextParser + _, err = parser.TextToMetricFamilies(strings.NewReader(strings.ReplaceAll(respData, "\r\n", "\n"))) + return err +} + +func fetchVersion(endpoint string, httpVersion string, connType clientConnType) error { + req := cURLReq{endpoint: "/version", timeout: 5, httpVersion: httpVersion} + respData, err := curl(endpoint, "GET", req, connType) + if err != nil { + return err + } + var resp version.Versions + return json.Unmarshal([]byte(respData), &resp) +} + +func fetchHealth(endpoint string, httpVersion string, connType clientConnType) error { + req := cURLReq{endpoint: "/health", timeout: 5, httpVersion: httpVersion} + respData, err := curl(endpoint, "GET", req, connType) + if err != nil { + return err + } + var resp etcdhttp.Health + return json.Unmarshal([]byte(respData), &resp) +} + +func fetchDebugVars(endpoint string, httpVersion string, connType clientConnType) error { + req := cURLReq{endpoint: "/debug/vars", timeout: 5, httpVersion: httpVersion} + respData, err := curl(endpoint, "GET", req, connType) + if err != nil { + return err + } + var resp map[string]interface{} + return json.Unmarshal([]byte(respData), &resp) +} + +func curl(endpoint string, method string, curlReq cURLReq, connType clientConnType) (string, error) { + args := cURLPrefixArgs(endpoint, connType, false, method, curlReq) + lines, err := runUtilCompletion(args, nil) + if err != nil { + return "", err + } + return strings.Join(lines, "\n"), nil +} diff --git a/tests/go.mod b/tests/go.mod index 5a452319e..d5cc7fd8c 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -23,6 +23,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/prometheus/client_golang v1.11.1 + github.com/prometheus/common v0.26.0 github.com/soheilhy/cmux v0.1.5 github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 @@ -63,7 +64,6 @@ require ( github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/sirupsen/logrus v1.7.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect