diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md index 2d0294cf2..7367557fc 100644 --- a/CHANGELOG/CHANGELOG-3.6.md +++ b/CHANGELOG/CHANGELOG-3.6.md @@ -78,6 +78,10 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0). - Fix [Restrict the max size of each WAL entry to the remaining size of the WAL file](https://github.com/etcd-io/etcd/pull/14122). - Fix [memberID equals zero in corruption alarm](https://github.com/etcd-io/etcd/pull/14272) +### etcd grpc-proxy + +- Add [`etcd grpc-proxy start --endpoints-auto-sync-interval`](https://github.com/etcd-io/etcd/pull/14354) flag to enable and configure interval of auto sync of endpoints with server. + ### tools/benchmark - [Add etcd client autoSync flag](https://github.com/etcd-io/etcd/pull/13416) diff --git a/server/etcdmain/grpc_proxy.go b/server/etcdmain/grpc_proxy.go index aab2d6f83..1bd09d0fb 100644 --- a/server/etcdmain/grpc_proxy.go +++ b/server/etcdmain/grpc_proxy.go @@ -57,15 +57,16 @@ import ( ) var ( - grpcProxyListenAddr string - grpcProxyMetricsListenAddr string - grpcProxyEndpoints []string - grpcProxyDNSCluster string - grpcProxyDNSClusterServiceName string - grpcProxyInsecureDiscovery bool - grpcProxyDataDir string - grpcMaxCallSendMsgSize int - grpcMaxCallRecvMsgSize int + grpcProxyListenAddr string + grpcProxyMetricsListenAddr string + grpcProxyEndpoints []string + grpcProxyEndpointsAutoSyncInterval time.Duration + grpcProxyDNSCluster string + grpcProxyDNSClusterServiceName string + grpcProxyInsecureDiscovery bool + grpcProxyDataDir string + grpcMaxCallSendMsgSize int + grpcMaxCallRecvMsgSize int // tls for connecting to etcd @@ -134,6 +135,7 @@ func newGRPCProxyStartCommand() *cobra.Command { cmd.Flags().StringVar(&grpcProxyMetricsListenAddr, "metrics-addr", "", "listen for endpoint /metrics requests on an additional interface") cmd.Flags().BoolVar(&grpcProxyInsecureDiscovery, "insecure-discovery", false, "accept insecure SRV records") cmd.Flags().StringSliceVar(&grpcProxyEndpoints, "endpoints", []string{"127.0.0.1:2379"}, "comma separated etcd cluster endpoints") + cmd.Flags().DurationVar(&grpcProxyEndpointsAutoSyncInterval, "endpoints-auto-sync-interval", 0, "etcd endpoints auto sync interval (disabled by default)") cmd.Flags().StringVar(&grpcProxyAdvertiseClientURL, "advertise-client-url", "127.0.0.1:23790", "advertise address to register (must be reachable by client)") cmd.Flags().StringVar(&grpcProxyResolverPrefix, "resolver-prefix", "", "prefix to use for registering proxy (must be shared with other grpc-proxy members)") cmd.Flags().IntVar(&grpcProxyResolverTTL, "resolver-ttl", 0, "specify TTL, in seconds, when registering proxy endpoints") @@ -338,8 +340,9 @@ func newProxyClientCfg(lg *zap.Logger, eps []string, tls *transport.TLSInfo) (*c func newClientCfg(lg *zap.Logger, eps []string) (*clientv3.Config, error) { // set tls if any one tls option set cfg := clientv3.Config{ - Endpoints: eps, - DialTimeout: 5 * time.Second, + Endpoints: eps, + AutoSyncInterval: grpcProxyEndpointsAutoSyncInterval, + DialTimeout: 5 * time.Second, } if grpcMaxCallSendMsgSize > 0 { diff --git a/tests/e2e/etcd_grpcproxy_test.go b/tests/e2e/etcd_grpcproxy_test.go new file mode 100644 index 000000000..b9c40daa8 --- /dev/null +++ b/tests/e2e/etcd_grpcproxy_test.go @@ -0,0 +1,169 @@ +// Copyright 2017 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/pkg/v3/expect" + "go.etcd.io/etcd/tests/v3/framework/config" + "go.etcd.io/etcd/tests/v3/framework/e2e" + "go.etcd.io/etcd/tests/v3/framework/testutils" +) + +func TestGrpcProxyAutoSync(t *testing.T) { + e2e.SkipInShortMode(t) + + var ( + node1Name = "node1" + node1ClientURL = "http://localhost:12379" + node1PeerURL = "http://localhost:12380" + + node2Name = "node2" + node2ClientURL = "http://localhost:22379" + node2PeerURL = "http://localhost:22380" + + proxyClientURL = "127.0.0.1:32379" + + autoSyncInterval = 1 * time.Second + ) + + // Run cluster of one node + proc1, err := runEtcdNode( + node1Name, t.TempDir(), + node1ClientURL, node1PeerURL, + "new", fmt.Sprintf("%s=%s", node1Name, node1PeerURL), + ) + require.NoError(t, err) + + // Run grpc-proxy instance + proxyProc, err := e2e.SpawnCmd([]string{e2e.BinDir + "/etcd", "grpc-proxy", "start", + "--advertise-client-url", proxyClientURL, "--listen-addr", proxyClientURL, + "--endpoints", node1ClientURL, + "--endpoints-auto-sync-interval", autoSyncInterval.String(), + }, nil) + require.NoError(t, err) + + proxyCtl := e2e.NewEtcdctl(&e2e.EtcdProcessClusterConfig{}, []string{proxyClientURL}) + err = proxyCtl.Put("k1", "v1", config.PutOptions{}) + require.NoError(t, err) + + memberCtl := e2e.NewEtcdctl(&e2e.EtcdProcessClusterConfig{}, []string{node1ClientURL}) + _, err = memberCtl.MemberAdd(node2Name, []string{node2PeerURL}) + if err != nil { + t.Fatal(err) + } + + // Run new member + proc2, err := runEtcdNode( + node2Name, t.TempDir(), + node2ClientURL, node2PeerURL, + "existing", fmt.Sprintf("%s=%s,%s=%s", node1Name, node1PeerURL, node2Name, node2PeerURL), + ) + require.NoError(t, err) + + // Wait for auto sync of endpoints + err = waitForEndpointInLog(proxyProc, node2ClientURL) + require.NoError(t, err) + + memberList, err := memberCtl.MemberList() + require.NoError(t, err) + + node1MemberID, err := findMemberIDByEndpoint(memberList.Members, node1ClientURL) + require.NoError(t, err) + + // Second node could be not ready yet + for i := 0; i < 10; i++ { + _, err = memberCtl.MemberRemove(node1MemberID) + if err != nil && strings.Contains(err.Error(), rpctypes.ErrGRPCUnhealthy.Error()) { + time.Sleep(500 * time.Millisecond) + continue + } + break + } + + // Remove node1 from member list and stop this nod + require.NoError(t, err) + require.NoError(t, proc1.Stop()) + + var resp *clientv3.GetResponse + for i := 0; i < 10; i++ { + resp, err = proxyCtl.Get("k1", config.GetOptions{}) + if err != nil && strings.Contains(err.Error(), rpctypes.ErrGRPCLeaderChanged.Error()) { + time.Sleep(500 * time.Millisecond) + continue + } + } + require.NoError(t, err) + kvs := testutils.KeyValuesFromGetResponse(resp) + assert.Equal(t, []testutils.KV{{Key: "k1", Val: "v1"}}, kvs) + + require.NoError(t, proc2.Stop()) + require.NoError(t, proxyProc.Stop()) +} + +func runEtcdNode(name, dataDir, clientURL, peerURL, clusterState, initialCluster string) (*expect.ExpectProcess, error) { + proc, err := e2e.SpawnCmd([]string{e2e.BinDir + "/etcd", + "--name", name, + "--data-dir", dataDir, + "--listen-client-urls", clientURL, "--advertise-client-urls", clientURL, + "--listen-peer-urls", peerURL, "--initial-advertise-peer-urls", peerURL, + "--initial-cluster-token", "etcd-cluster", + "--initial-cluster-state", clusterState, + "--initial-cluster", initialCluster, + }, nil) + if err != nil { + return nil, err + } + + _, err = proc.Expect("ready to serve client requests") + + return proc, err +} + +func findMemberIDByEndpoint(members []*etcdserverpb.Member, endpoint string) (uint64, error) { + for _, m := range members { + if m.ClientURLs[0] == endpoint { + return m.ID, nil + } + } + + return 0, fmt.Errorf("member not found") +} + +func waitForEndpointInLog(proxyProc *expect.ExpectProcess, endpoint string) error { + endpoint = strings.Replace(endpoint, "http://", "", 1) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := proxyProc.ExpectFunc(ctx, func(s string) bool { + if strings.Contains(s, endpoint) && strings.Contains(s, "Resolver state updated") { + return true + } + return false + }) + + return err +} diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index d6376c9fc..fd5e7d81d 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -238,6 +238,12 @@ func (ctl *EtcdctlV3) MemberList() (*clientv3.MemberListResponse, error) { return &resp, err } +func (ctl *EtcdctlV3) MemberAdd(name string, peerAddrs []string) (*clientv3.MemberAddResponse, error) { + var resp clientv3.MemberAddResponse + err := ctl.spawnJsonCmd(&resp, "member", "add", name, "--peer-urls", strings.Join(peerAddrs, ",")) + return &resp, err +} + func (ctl *EtcdctlV3) MemberAddAsLearner(name string, peerAddrs []string) (*clientv3.MemberAddResponse, error) { var resp clientv3.MemberAddResponse err := ctl.spawnJsonCmd(&resp, "member", "add", name, "--learner", "--peer-urls", strings.Join(peerAddrs, ","))