diff --git a/integration/cluster_test.go b/integration/cluster_test.go index ca58e2d9a..4232a56e5 100644 --- a/integration/cluster_test.go +++ b/integration/cluster_test.go @@ -72,6 +72,14 @@ func testCluster(t *testing.T, size int) { clusterMustProgress(t, c.Members) } +func TestTLSClusterOf3(t *testing.T) { + defer afterTest(t) + c := NewTLSCluster(t, 3) + c.Launch(t) + defer c.Terminate(t) + clusterMustProgress(t, c.Members) +} + func TestClusterOf1UsingDiscovery(t *testing.T) { testClusterUsingDiscovery(t, 1) } func TestClusterOf3UsingDiscovery(t *testing.T) { testClusterUsingDiscovery(t, 3) } @@ -203,8 +211,12 @@ type cluster struct { func fillClusterForMembers(ms []*member, cName string) error { addrs := make([]string, 0) for _, m := range ms { + scheme := "http" + if !m.PeerTLSInfo.Empty() { + scheme = "https" + } for _, l := range m.PeerListeners { - addrs = append(addrs, fmt.Sprintf("%s=%s", m.Name, "http://"+l.Addr().String())) + addrs = append(addrs, fmt.Sprintf("%s=%s://%s", m.Name, scheme, l.Addr().String())) } } clusterStr := strings.Join(addrs, ",") @@ -218,13 +230,11 @@ func fillClusterForMembers(ms []*member, cName string) error { return nil } -// NewCluster returns an unlaunched cluster of the given size which has been -// set to use static bootstrap. -func NewCluster(t *testing.T, size int) *cluster { +func newCluster(t *testing.T, size int, usePeerTLS bool) *cluster { c := &cluster{} ms := make([]*member, size) for i := 0; i < size; i++ { - ms[i] = mustNewMember(t, c.name(i)) + ms[i] = mustNewMember(t, c.name(i), usePeerTLS) } c.Members = ms if err := fillClusterForMembers(c.Members, clusterName); err != nil { @@ -234,19 +244,29 @@ func NewCluster(t *testing.T, size int) *cluster { return c } +// NewCluster returns an unlaunched cluster of the given size which has been +// set to use static bootstrap. +func NewCluster(t *testing.T, size int) *cluster { + return newCluster(t, size, false) +} + // NewClusterUsingDiscovery returns an unlaunched cluster of the given size // which has been set to use the given url as discovery service to bootstrap. func NewClusterByDiscovery(t *testing.T, size int, url string) *cluster { c := &cluster{} ms := make([]*member, size) for i := 0; i < size; i++ { - ms[i] = mustNewMember(t, c.name(i)) + ms[i] = mustNewMember(t, c.name(i), false) ms[i].DiscoveryURL = url } c.Members = ms return c } +func NewTLSCluster(t *testing.T, size int) *cluster { + return newCluster(t, size, true) +} + func (c *cluster) Launch(t *testing.T) { errc := make(chan error) for _, m := range c.Members { @@ -282,9 +302,13 @@ func (c *cluster) URLs() []string { func (c *cluster) HTTPMembers() []client.Member { ms := make([]client.Member, len(c.Members)) for i, m := range c.Members { + scheme := "http" + if !m.PeerTLSInfo.Empty() { + scheme = "https" + } ms[i].Name = m.Name for _, ln := range m.PeerListeners { - ms[i].PeerURLs = append(ms[i].PeerURLs, "http://"+ln.Addr().String()) + ms[i].PeerURLs = append(ms[i].PeerURLs, scheme+"://"+ln.Addr().String()) } for _, ln := range m.ClientListeners { ms[i].ClientURLs = append(ms[i].ClientURLs, "http://"+ln.Addr().String()) @@ -296,7 +320,8 @@ func (c *cluster) HTTPMembers() []client.Member { func (c *cluster) AddMember(t *testing.T) { clusterStr := c.Members[0].Cluster.String() idx := len(c.Members) - m := mustNewMember(t, c.name(idx)) + // TODO: support add TLS member + m := mustNewMember(t, c.name(idx), false) // send add request to the cluster cc := mustNewHTTPClient(t, []string{c.URL(0)}) @@ -445,22 +470,43 @@ func newListenerWithAddr(t *testing.T, addr string) net.Listener { type member struct { etcdserver.ServerConfig PeerListeners, ClientListeners []net.Listener + // inited PeerTLSInfo implies to enable peer TLS + PeerTLSInfo transport.TLSInfo raftHandler *testutil.PauseableHandler s *etcdserver.EtcdServer hss []*httptest.Server } -func mustNewMember(t *testing.T, name string) *member { - var err error +// mustNewMember return an inited member with the given name. If usePeerTLS is +// true, it will set PeerTLSInfo and use https scheme to communicate between +// peers. +func mustNewMember(t *testing.T, name string, usePeerTLS bool) *member { + var ( + testTLSInfo = transport.TLSInfo{ + KeyFile: "./fixtures/server.key.insecure", + CertFile: "./fixtures/server.crt", + TrustedCAFile: "./fixtures/ca.crt", + ClientCertAuth: true, + } + err error + ) m := &member{} + peerScheme := "http" + if usePeerTLS { + peerScheme = "https" + } + pln := newLocalListener(t) m.PeerListeners = []net.Listener{pln} - m.PeerURLs, err = types.NewURLs([]string{"http://" + pln.Addr().String()}) + m.PeerURLs, err = types.NewURLs([]string{peerScheme + "://" + pln.Addr().String()}) if err != nil { t.Fatal(err) } + if usePeerTLS { + m.PeerTLSInfo = testTLSInfo + } cln := newLocalListener(t) m.ClientListeners = []net.Listener{cln} @@ -475,13 +521,13 @@ func mustNewMember(t *testing.T, name string) *member { if err != nil { t.Fatal(err) } - clusterStr := fmt.Sprintf("%s=http://%s", name, pln.Addr().String()) + clusterStr := fmt.Sprintf("%s=%s://%s", name, peerScheme, pln.Addr().String()) m.Cluster, err = etcdserver.NewClusterFromString(clusterName, clusterStr) if err != nil { t.Fatal(err) } m.NewCluster = true - m.Transport = mustNewTransport(t) + m.Transport = mustNewTransport(t, m.PeerTLSInfo) m.ElectionTicks = electionTicks m.TickMs = uint(tickDuration / time.Millisecond) return m @@ -512,8 +558,9 @@ func (m *member) Clone(t *testing.T) *member { // this should never fail panic(err) } - mm.Transport = mustNewTransport(t) + mm.Transport = mustNewTransport(t, m.PeerTLSInfo) mm.ElectionTicks = m.ElectionTicks + mm.PeerTLSInfo = m.PeerTLSInfo return mm } @@ -534,7 +581,15 @@ func (m *member) Launch() error { Listener: ln, Config: &http.Server{Handler: m.raftHandler}, } - hs.Start() + if m.PeerTLSInfo.Empty() { + hs.Start() + } else { + hs.TLS, err = m.PeerTLSInfo.ServerConfig() + if err != nil { + return err + } + hs.StartTLS() + } m.hss = append(m.hss, hs) } for _, ln := range m.ClientListeners { @@ -616,7 +671,7 @@ func (m *member) Terminate(t *testing.T) { } func mustNewHTTPClient(t *testing.T, eps []string) client.Client { - cfg := client.Config{Transport: mustNewTransport(t), Endpoints: eps} + cfg := client.Config{Transport: mustNewTransport(t, transport.TLSInfo{}), Endpoints: eps} c, err := client.New(cfg) if err != nil { t.Fatal(err) @@ -624,8 +679,8 @@ func mustNewHTTPClient(t *testing.T, eps []string) client.Client { return c } -func mustNewTransport(t *testing.T) *http.Transport { - tr, err := transport.NewTimeoutTransport(transport.TLSInfo{}, rafthttp.DialTimeout, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) +func mustNewTransport(t *testing.T, tlsInfo transport.TLSInfo) *http.Transport { + tr, err := transport.NewTimeoutTransport(tlsInfo, rafthttp.DialTimeout, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout) if err != nil { t.Fatal(err) } diff --git a/integration/fixtures/ca.crt b/integration/fixtures/ca.crt new file mode 100644 index 000000000..eeba59685 --- /dev/null +++ b/integration/fixtures/ca.crt @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFNDCCAx6gAwIBAgIBATALBgkqhkiG9w0BAQswLTEMMAoGA1UEBhMDVVNBMRAw +DgYDVQQKEwdldGNkLWNhMQswCQYDVQQLEwJDQTAeFw0xNTAzMTMxNzUxNTJaFw0y +NTAzMTMxNzUxNThaMC0xDDAKBgNVBAYTA1VTQTEQMA4GA1UEChMHZXRjZC1jYTEL +MAkGA1UECxMCQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC2ENR3 +ic04SAtr1qnGJv1p8VMkYln+Ppjxh9uP3Jb7v6zNv5MzUA3tIHGhDL2FdLy/hvaK +/uXz4kIPh6i+8Icua/Nmyyeyvnv5x4BGR90svo/wkUnXCufzKEjzTpU3O3O/TrBQ +f04rF9hAsi/6T/KytU+u8Xm4BrONP4yBKCviSag+VwB1h9LRQWgWcWM6ujxpT0ma +DLan56vRlZIRviO8cJqQhVkmkXwKMl1fQ2RU4IvOgHliRrYjdfjGkjQ9cXf5di0w +jufKX5CVJvcfagZyew4bMypsz/8SuK65qjmsH00WwyVw+2iD1smtsp3M1SiT6+Qr +otsdVl3E0CQsunGg7vb9Lw9iH4bjMwqZUEJ+FSKAXrYI7xScsP2XXU0EeJiHuJF/ +U7eDaBJTKnlSWxub7D3eong+s9IKTYUpdgxS6DFs3sY++gbCCN0d2OOy1jfhsoGl +wIq2A5IkNlSXCQHJ7VRz36R5qkPFL8LWui80SgzthvSOUX8MhNXd6czYUGpYNBYM +wCJfZaCxuhgxgipxm1sipukaoMSLXgvXY7CRlo9uP6pHHUex79iq4BkeYtTyw6ZJ +HpLHomlX7f3q4h3rYeRyE0+BfkWpuwX9AuJu/zFBKGx7Llsagm7kmzA6FsPjF+2I +iS5vYIVkGi1AGztqpue07eeqjUC0ZtnlnZ1j+wIDAQABo2MwYTAOBgNVHQ8BAf8E +BAMCAAQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU4I9pFzO3FIpFHSdZuFJX +5Xwyx6owHwYDVR0jBBgwFoAU4I9pFzO3FIpFHSdZuFJX5Xwyx6owCwYJKoZIhvcN +AQELA4ICAQAfLGNhTEQ5jOmCwQwZaB9Wg7hRIbYbdoB7vl9hIXwvdDEqmYNM+Msg +wWKGCFnIiLwMigizfipDwY9fzcVGunKdlz8wmZpKtMnLnXura7AjC5qEq9Hpidmq +EZ4fG7xQUToUJThCp4TrvFJNU+69/SeGlPLs9UYdlq2/MBlv17QsM13ZYwR00WCj +5Z+eoP/UHBgQc+QvgQxvQitgqmXudf9Y1+YD2j5wo7aqI1Y1sIlZMCeLt7U1XwMf +ndwv+ZvcB1Vir7bWc1JN/L2lh0knqmZq0+PMFVwM6BHnQmfQmSTjmEvSlupJ5AJu +cVKAQkfOOhTd1v7gvovn2zFmIcN4Fj7ZCsHSvNEGiBQb4YecG7+R2lO86RM6PB/C +pz8526kvK70J0d5CiE0WzuyaP4rxQXLBYo6uZCredVbazLzfZVg/3pg4xIOUDO0c +j25ofWX/BOujLiYcasx4Pje54gBGmVLt2Py6ZGpkFGkZmqlv3lsCM2UgxbkYVkRr +vIj2YoZvgWm6OgpwJXjjsMc9Kvm3Q1qCkMOpHXn9wSzTlfpc1fliuXRlBMFxHDRs +C5iLeQKT5UVXuZ8b6EENYRW+RD4Y0EIC9lzHqoD86TwQywZeJ9KeP+PisTASSEjT +USW9UvCoGfDGvnK3Niw61KqMWXf/7a5wc1h/Y62u+pHf1TPJFzdnew== +-----END CERTIFICATE----- diff --git a/integration/fixtures/server.crt b/integration/fixtures/server.crt new file mode 100644 index 000000000..454631cc8 --- /dev/null +++ b/integration/fixtures/server.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDnzCCAYmgAwIBAgIBAjALBgkqhkiG9w0BAQswLTEMMAoGA1UEBhMDVVNBMRAw +DgYDVQQKEwdldGNkLWNhMQswCQYDVQQLEwJDQTAeFw0xNTAzMTMxNzUxNThaFw0y +NTAzMTMxNzUxNTlaMEYxDDAKBgNVBAYTA1VTQTEQMA4GA1UEChMHZXRjZC1jYTEQ +MA4GA1UECxMHc2VydmVyMTESMBAGA1UEAxMJbG9jYWxob3N0MFwwDQYJKoZIhvcN +AQEBBQADSwAwSAJBAMG0aSZGVR/MDP5Nsn6Xj2dUADDCiGDcmzvaE4uFJ3L4zr+A +4ReyfdlQfnfVlHRRkbJtGTzdgYnunU4KQWKJrIcCAwEAAaN9MHswHQYDVR0lBBYw +FAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBSk5l0p37da8p2XSfx8H9eK +oMG3eDAfBgNVHSMEGDAWgBTgj2kXM7cUikUdJ1m4UlflfDLHqjAaBgNVHREEEzAR +gglsb2NhbGhvc3SHBH8AAAEwCwYJKoZIhvcNAQELA4ICAQAE0pWMwhl0myXwdMQ5 +hwiOnTQ5ztaldVAzVb/dDY3XkqTpMSSteJlDQ+CcltRCmB81z2Rpm41Jh87BjWXR +FQO0kHdba9uo6czjUhFfU1e7fPIB8g2+4VXDKhnto5kdvg2t5bcowfWSsIc+w/Mw +vHN4a4xGAksz24jpzVkz3zLplYx6KO/VKKTWJjZRKl6X2VcYOJtgwX3s1bTQ/8Ng +1uDwun07Ng62WO4Ym4ceMG7qDIDK13K3F9/knQErXYUuQtKwCVKMfrZTWVaeV2K3 +q6AiN7LOqlauZd7t7Bnhn20kd6nd76HRSC/kvw7FmTDE9bHQrgN4pAi0vMiSnbll +rw8SGwGIPrlgV8+s0ttEibIryzSO3SWiNgLqRotWZmNre+eKoEP7USFTJ8DCj5r+ +o3Aj96zN+fHq81sklQTARht9IIUJbok8vFBBPNwZu1k3U1284osbsSYEL98I88YA +BCpn+R9QHp+q3L4qq8GIEbCOUC5akZ6w5U86nW2NV2rvU9TwgEmE5/SX0m30jYJE +Ou39H2LsOI/k/HO/mp0uu5U0AkzRsE6I8t0JtOk4JFmf0EeMjEPBFxI2oNBTT4ML +ZLwS5IMxIOI/f6rCp8129IQzkLATBsX9d+m0l5NVjyJJullEOwXwGqtP1PJgN9xM +yY049unGp1SAGRCbkl5Jbxu1Mg== +-----END CERTIFICATE----- diff --git a/integration/fixtures/server.key.insecure b/integration/fixtures/server.key.insecure new file mode 100644 index 000000000..70b7fb8dc --- /dev/null +++ b/integration/fixtures/server.key.insecure @@ -0,0 +1,9 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIBOgIBAAJBAMG0aSZGVR/MDP5Nsn6Xj2dUADDCiGDcmzvaE4uFJ3L4zr+A4Rey +fdlQfnfVlHRRkbJtGTzdgYnunU4KQWKJrIcCAwEAAQJATa42BN1rwcao50rtbAsH +vV5YfTVmT8HolwYLlIYfneEpapAM6j1F4d0lI4hE7oQ3ACdTg/KHLSsfLk1qtxsz +gQIhAP+SXbIF8L949XFSf0bhzhvTozbQ3T0+VN2+tohF6eSnAiEAwgeBXDQ8gwax +rHR5iiaO3I0VAagya2CPKqwOQ2wClSECICCL6Ta0nvYjV055LRA/zVmp2A0xCBu2 +hmQ+10v/a0vdAiEAoQ3Xy7A0VlI1Ir/frtPIm7ujpyd1Cnow/Cuq/z3leuECIHs3 +pAt/kupzaWBdB711tBwCdd+iQag0jGuN8Shk6A0U +-----END RSA PRIVATE KEY----- diff --git a/integration/member_test.go b/integration/member_test.go index 96d09dee3..f49f51498 100644 --- a/integration/member_test.go +++ b/integration/member_test.go @@ -82,7 +82,7 @@ func TestLaunchDuplicateMemberShouldFail(t *testing.T) { func TestSnapshotAndRestartMember(t *testing.T) { defer afterTest(t) - m := mustNewMember(t, "snapAndRestartTest") + m := mustNewMember(t, "snapAndRestartTest", false) m.SnapCount = 100 m.Launch() defer m.Terminate(t) diff --git a/integration/migration_test.go b/integration/migration_test.go index b8e5add65..95fc3e988 100644 --- a/integration/migration_test.go +++ b/integration/migration_test.go @@ -21,7 +21,7 @@ import ( func TestUpgradeMember(t *testing.T) { defer afterTest(t) - m := mustNewMember(t, "integration046") + m := mustNewMember(t, "integration046", false) cmd := exec.Command("cp", "-r", "testdata/integration046_data/conf", "testdata/integration046_data/log", "testdata/integration046_data/snapshot", m.DataDir) err := cmd.Run() if err != nil {