Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement flag --experimental-set-member-localaddr

Browse Source 
Which sets the LocalAddr to an IP address from --initial-advertise-peer-urls.

Also adds e2e test that requires this flag to succeed.

Co-authored-by: HighPon <s.shiraki.business@gmail.com>
Signed-off-by: Edwin Xie <edwin.xie@broadcom.com>
This commit is contained in:
Edwin Xie 2024-05-24 18:16:50 +00:00
parent 7a3ca53345
commit 4c77726914
10 changed files with 435 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG/CHANGELOG-3.6.md
View File

@ -75,6 +75,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0).
- Decreased [`--snapshot-count` default value from 100,000 to 10,000](https://github.com/etcd-io/etcd/pull/15408)
- Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15156) to enable support for TLS 1.3.
- Add [quota to endpoint status response](https://github.com/etcd-io/etcd/pull/17877)
- Add ['etcd --experimental-set-member-localaddr'](https://github.com/etcd-io/etcd/pull/17661) to enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer.
### etcd grpc-proxy

3
client/pkg/transport/listener.go
View File

@ -193,6 +193,9 @@ type TLSInfo struct {
// EmptyCN indicates that the cert must have empty CN.
// If true, ClientConfig() will return an error for a cert with non empty CN.
EmptyCN bool
// LocalAddr is the local IP address to use when communicating with a peer.
LocalAddr string
}
func (info TLSInfo) String() string {

11
client/pkg/transport/transport.go
View File

@ -30,10 +30,19 @@ func NewTransport(info TLSInfo, dialtimeoutd time.Duration) (*http.Transport, er
return nil, err
}
var ipAddr net.Addr
if info.LocalAddr != "" {
ipAddr, err = net.ResolveTCPAddr("tcp", info.LocalAddr+":0")
if err != nil {
return nil, err
}
}
t := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: dialtimeoutd,
Timeout: dialtimeoutd,
LocalAddr: ipAddr,
// value taken from http.DefaultTransport
KeepAlive: 30 * time.Second,
}).DialContext,

3
server/config/config.go
View File

@ -204,6 +204,9 @@ type ServerConfig struct {
// V2Deprecation defines a phase of v2store deprecation process.
V2Deprecation V2DeprecationEnum `json:"v2-deprecation"`
// ExperimentalLocalAddress is the local IP address to use when communicating with a peer.
ExperimentalLocalAddress string `json:"experimental-local-address"`
}
// VerifyBootstrap sanity-checks the initial config for bootstrap case

43
server/embed/config.go
View File

@ -22,6 +22,7 @@ import (
"math"
"net"
"net/http"
"net/netip"
"net/url"
"os"
"path/filepath"
@ -225,6 +226,12 @@ type Config struct {
ClientAutoTLS bool
PeerTLSInfo transport.TLSInfo
PeerAutoTLS bool
// ExperimentalSetMemberLocalAddr enables using the first specified and
// non-loopback local address from initial-advertise-peer-urls as the local
// address when communicating with a peer.
ExperimentalSetMemberLocalAddr bool `json:"experimental-set-member-localaddr"`
// SelfSignedCertValidity specifies the validity period of the client and peer certificates
// that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS,
// the unit is year, and the default is 1
@ -621,6 +628,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
"initial-advertise-peer-urls",
"List of this member's peer URLs to advertise to the rest of the cluster.",
)
fs.BoolVar(&cfg.ExperimentalSetMemberLocalAddr, "experimental-set-member-localaddr", false, "Enable to have etcd use the first specified and non-loopback host from initial-advertise-peer-urls as the local address when communicating with a peer.")
fs.Var(
flags.NewUniqueURLsWithExceptions(DefaultAdvertiseClientURLs, ""),
"advertise-client-urls",
@ -1148,6 +1157,40 @@ func (cfg *Config) InitialClusterFromName(name string) (ret string) {
return ret[1:]
}
// InferLocalAddr tries to determine the LocalAddr used when communicating with
// an etcd peer. If SetMemberLocalAddr is true, then it will try to get the host
// from AdvertisePeerUrls by searching for the first URL with a specified
// non-loopback address. Otherwise, it defaults to empty string and the
// LocalAddr used will be the default for the Golang HTTP client.
func (cfg *Config) InferLocalAddr() string {
if !cfg.ExperimentalSetMemberLocalAddr {
return ""
}
lg := cfg.GetLogger()
lg.Info(
"searching for a suitable member local address in AdvertisePeerURLs",
zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()),
)
for _, peerURL := range cfg.AdvertisePeerUrls {
if addr, err := netip.ParseAddr(peerURL.Hostname()); err == nil {
if addr.IsLoopback() || addr.IsUnspecified() {
continue
}
lg.Info(
"setting member local address",
zap.String("LocalAddr", addr.String()),
)
return addr.String()
}
}
lg.Warn(
"unable to set a member local address due to lack of suitable local addresses",
zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()),
)
return ""
}
func (cfg *Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
func (cfg *Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }

132
server/embed/config_test.go
View File

@ -153,6 +153,138 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) {
}
}
func TestInferLocalAddr(t *testing.T) {
tests := []struct {
name string
advertisePeerURLs []string
setMemberLocalAddr bool
expectedLocalAddr string
}{
{
"defaults, ExperimentalSetMemberLocalAddr=false ",
[]string{DefaultInitialAdvertisePeerURLs},
false,
"",
},
{
"IPv4 address, ExperimentalSetMemberLocalAddr=false ",
[]string{"https://192.168.100.110:2380"},
false,
"",
},
{
"defaults, ExperimentalSetMemberLocalAddr=true",
[]string{DefaultInitialAdvertisePeerURLs},
true,
"",
},
{
"IPv4 unspecified address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://0.0.0.0:2380"},
true,
"",
},
{
"IPv6 unspecified address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://[::]:2380"},
true,
"",
},
{
"IPv4 loopback address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://127.0.0.1:2380"},
true,
"",
},
{
"IPv6 loopback address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://[::1]:2380"},
true,
"",
},
{
"IPv4 address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://192.168.100.110:2380"},
true,
"192.168.100.110",
},
{
"Hostname only, ExperimentalSetMemberLocalAddr=true",
[]string{"https://123-host-3.corp.internal:2380"},
true,
"",
},
{
"Hostname and IPv4 address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380"},
true,
"192.168.100.110",
},
{
"IPv4 address and Hostname, ExperimentalSetMemberLocalAddr=true",
[]string{"https://192.168.100.110:2380", "https://123-host-3.corp.internal:2380"},
true,
"192.168.100.110",
},
{
"IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true",
[]string{"https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
true,
"192.168.100.110",
},
{
"IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true",
// IPv4 addresses will always sort before IPv6 ones anyway
[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"},
true,
"192.168.100.110",
},
{
"Hostname, IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true",
[]string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
true,
"192.168.100.110",
},
{
"Hostname, IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true",
// IPv4 addresses will always sort before IPv6 ones anyway
[]string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"},
true,
"192.168.100.110",
},
{
"IPv6 address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380"},
true,
"2001:db8:85a3::8a2e:370:7334",
},
{
"Hostname and IPv6 address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
true,
"2001:db8:85a3::8a2e:370:7334",
},
{
"IPv6 address and Hostname, ExperimentalSetMemberLocalAddr=true",
[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://123-host-3.corp.internal:2380"},
true,
"2001:db8:85a3::8a2e:370:7334",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := NewConfig()
cfg.AdvertisePeerUrls = types.MustNewURLs(tt.advertisePeerURLs)
cfg.ExperimentalSetMemberLocalAddr = tt.setMemberLocalAddr
require.NoError(t, cfg.Validate())
require.Equal(t, tt.expectedLocalAddr, cfg.InferLocalAddr())
})
}
}
func (s *securityConfig) equals(t *transport.TLSInfo) bool {
return s.CertFile == t.CertFile &&
s.CertAuth == t.ClientCertAuth &&

5
server/embed/etcd.go
View File

@ -227,6 +227,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes,
ExperimentalMaxLearners: cfg.ExperimentalMaxLearners,
V2Deprecation: cfg.V2DeprecationEffective(),
ExperimentalLocalAddress: cfg.InferLocalAddr(),
}
if srvcfg.ExperimentalEnableDistributedTracing {
@ -245,6 +246,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
)
}
srvcfg.PeerTLSInfo.LocalAddr = srvcfg.ExperimentalLocalAddress
print(e.cfg.logger, *cfg, srvcfg, memberInitialized)
if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
@ -336,6 +339,8 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized
zap.Strings("advertise-client-urls", ec.getAdvertiseClientURLs()),
zap.Strings("listen-client-urls", ec.getListenClientURLs()),
zap.Strings("listen-metrics-urls", ec.getMetricsURLs()),
zap.Bool("experimental-set-member-localaddr", ec.ExperimentalSetMemberLocalAddr),
zap.String("experimental-local-address", sc.ExperimentalLocalAddress),
zap.Strings("cors", cors),
zap.Strings("host-whitelist", hss),
zap.String("initial-cluster", sc.InitialPeerURLsMap.String()),

2
server/etcdmain/help.go
View File

@ -107,6 +107,8 @@ Member:
Clustering:
--initial-advertise-peer-urls 'http://localhost:2380'
List of this member's peer URLs to advertise to the rest of the cluster.
--experimental-set-member-localaddr 'false'
Enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer.
--initial-cluster 'default=http://localhost:2380'
Initial cluster configuration for bootstrapping.
--initial-cluster-state 'new'

110
tests/e2e/etcd_config_test.go
View File

@ -17,6 +17,7 @@ package e2e
import (
"context"
"fmt"
"net"
"os"
"strings"
"testing"
@ -274,6 +275,115 @@ func TestEtcdPeerNameAuth(t *testing.T) {
}
}
// TestEtcdPeerLocalAddr checks that the inter peer auth works with when
// the member LocalAddr is set.
func TestEtcdPeerLocalAddr(t *testing.T) {
e2e.SkipInShortMode(t)
nodeIP, err := getLocalIP()
t.Log("Using node IP", nodeIP)
require.NoError(t, err)
peers, tmpdirs := make([]string, 3), make([]string, 3)
for i := range peers {
peerIP := nodeIP
if i == 0 {
peerIP = "127.0.0.1"
}
peers[i] = fmt.Sprintf("e%d=https://%s:%d", i, peerIP, e2e.EtcdProcessBasePort+i)
tmpdirs[i] = t.TempDir()
}
procs := make([]*expect.ExpectProcess, len(peers))
defer func() {
for i := range procs {
if procs[i] != nil {
procs[i].Stop()
procs[i].Close()
}
os.RemoveAll(tmpdirs[i])
}
}()
tempDir := t.TempDir()
caFile, certFiles, keyFiles, err := generateCertsForIPs(tempDir, []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP(nodeIP)})
require.NoError(t, err)
defer func() {
os.RemoveAll(tempDir)
}()
// node 0 (127.0.0.1) does not set `--experimental-set-member-localaddr`,
// while nodes 1 and nodes 2 do.
//
// node 0's peer certificate is signed for 127.0.0.1, but it uses the host
// IP (by default) to communicate with peers, so they don't match.
// Accordingly, other peers will reject connections from node 0.
//
// Both node 1 and node 2's peer certificates are signed for the host IP,
// and they also communicate with peers using the host IP (explicitly set
// with --initial-advertise-peer-urls and
// --experimental-set-member-localaddr), so node 0 has no issue connecting
// to them.
//
// Refer to https://github.com/etcd-io/etcd/issues/17068.
for i := range procs {
peerIP := nodeIP
if i == 0 {
peerIP = "127.0.0.1"
}
ic := strings.Join(peers, ",")
commonArgs := []string{
e2e.BinPath.Etcd,
"--name", fmt.Sprintf("e%d", i),
"--listen-client-urls", "http://0.0.0.0:0",
"--data-dir", tmpdirs[i],
"--advertise-client-urls", "http://0.0.0.0:0",
"--initial-advertise-peer-urls", fmt.Sprintf("https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i),
"--listen-peer-urls", fmt.Sprintf("https://%s:%d,https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i, peerIP, e2e.EtcdProcessBasePort+len(peers)+i),
"--initial-cluster", ic,
}
var args []string
if i == 0 {
args = []string{
"--peer-cert-file", certFiles[0],
"--peer-key-file", keyFiles[0],
"--peer-trusted-ca-file", caFile,
"--peer-client-cert-auth",
}
} else {
args = []string{
"--peer-cert-file", certFiles[1],
"--peer-key-file", keyFiles[1],
"--peer-trusted-ca-file", caFile,
"--peer-client-cert-auth",
"--experimental-set-member-localaddr",
}
}
commonArgs = append(commonArgs, args...)
p, err := e2e.SpawnCmd(commonArgs, nil)
if err != nil {
t.Fatal(err)
}
procs[i] = p
}
for i, p := range procs {
var expect []string
if i == 0 {
expect = e2e.EtcdServerReadyLines
} else {
expect = []string{"x509: certificate is valid for 127.0.0.1, not "}
}
if err := e2e.WaitReadyExpectProc(context.TODO(), p, expect); err != nil {
t.Fatal(err)
}
}
}
func TestGrpcproxyAndCommonName(t *testing.T) {
e2e.SkipInShortMode(t)

126
tests/e2e/utils.go
View File

@ -15,9 +15,18 @@
package e2e
import (
"bytes"
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/json"
"encoding/pem"
"fmt"
"math/big"
"net"
"os"
"strings"
"testing"
"time"
@ -145,3 +154,120 @@ func patchArgs(args []string, flag, newValue string) error {
}
return fmt.Errorf("--%s flag not found", flag)
}
func generateCertsForIPs(tempDir string, ips []net.IP) (caFile string, certFiles []string, keyFiles []string, err error) {
ca := &x509.Certificate{
SerialNumber: big.NewInt(1001),
Subject: pkix.Name{
Organization: []string{"etcd"},
OrganizationalUnit: []string{"etcd Security"},
Locality: []string{"San Francisco"},
Province: []string{"California"},
Country: []string{"USA"},
},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(0, 0, 1),
IsCA: true,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
}
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return "", nil, nil, err
}
caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey)
if err != nil {
return "", nil, nil, err
}
caFile, _, err = saveCertToFile(tempDir, caBytes, nil)
if err != nil {
return "", nil, nil, err
}
for i, ip := range ips {
cert := &x509.Certificate{
SerialNumber: big.NewInt(1001 + int64(i)),
Subject: pkix.Name{
Organization: []string{"etcd"},
OrganizationalUnit: []string{"etcd Security"},
Locality: []string{"San Francisco"},
Province: []string{"California"},
Country: []string{"USA"},
},
IPAddresses: []net.IP{ip},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(0, 0, 1),
SubjectKeyId: []byte{1, 2, 3, 4, 5},
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature,
}
certKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return "", nil, nil, err
}
certBytes, err := x509.CreateCertificate(rand.Reader, cert, ca, &certKey.PublicKey, caKey)
if err != nil {
return "", nil, nil, err
}
certFile, keyFile, err := saveCertToFile(tempDir, certBytes, certKey)
if err != nil {
return "", nil, nil, err
}
certFiles = append(certFiles, certFile)
keyFiles = append(keyFiles, keyFile)
}
return caFile, certFiles, keyFiles, nil
}
func saveCertToFile(tempDir string, certBytes []byte, key *rsa.PrivateKey) (certFile string, keyFile string, err error) {
certPEM := new(bytes.Buffer)
pem.Encode(certPEM, &pem.Block{
Type: "CERTIFICATE",
Bytes: certBytes,
})
cf, err := os.CreateTemp(tempDir, "*.crt")
if err != nil {
return "", "", err
}
defer cf.Close()
if _, err := cf.Write(certPEM.Bytes()); err != nil {
return "", "", err
}
if key != nil {
certKeyPEM := new(bytes.Buffer)
pem.Encode(certKeyPEM, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(key),
})
kf, err := os.CreateTemp(tempDir, "*.key.insecure")
if err != nil {
return "", "", err
}
defer kf.Close()
if _, err := kf.Write(certKeyPEM.Bytes()); err != nil {
return "", "", err
}
return cf.Name(), kf.Name(), nil
}
return cf.Name(), "", nil
}
func getLocalIP() (string, error) {
conn, err := net.Dial("udp", "8.8.8.8:80")
if err != nil {
return "", err
}
defer conn.Close()
localAddress := conn.LocalAddr().(*net.UDPAddr)
return localAddress.IP.String(), nil
}