mirror of
https://github.com/etcd-io/etcd.git
synced 2024-09-27 06:25:44 +00:00
server: Add --listen-client-http-urls flag to allow running grpc server separate from http server
Difference in load configuration for watch delay tests show how huge the impact is. Even with random write scheduler grpc under http server can only handle 500 KB with 2 seconds delay. On the other hand, separate grpc server easily hits 10, 100 or even 1000 MB within 100 miliseconds. Priority write scheduler that was used in most previous releases is far worse than random one. Tests configured to only 5 MB to avoid flakes and taking too long to fill etcd. Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
This commit is contained in:
Marek Siarkowicz
@@ -56,6 +56,7 @@ type serveCtx struct {
|
||||
network string
|
||||
secure bool
|
||||
insecure bool
|
||||
httpOnly bool
|
||||
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
@@ -94,6 +95,7 @@ func (sctx *serveCtx) serve(
|
||||
handler http.Handler,
|
||||
errHandler func(error),
|
||||
grpcDialForRestGatewayBackends func(ctx context.Context) (*grpc.ClientConn, error),
|
||||
splitHttp bool,
|
||||
gopts ...grpc.ServerOption) (err error) {
|
||||
logger := defaultLog.New(ioutil.Discard, "etcdhttp", 0)
|
||||
<-s.ReadyNotify()
|
||||
@@ -101,6 +103,12 @@ func (sctx *serveCtx) serve(
|
||||
sctx.lg.Info("ready to serve client requests")
|
||||
|
||||
m := cmux.New(sctx.l)
|
||||
var server func() error
|
||||
onlyGRPC := splitHttp && !sctx.httpOnly
|
||||
onlyHttp := splitHttp && sctx.httpOnly
|
||||
grpcEnabled := !onlyHttp
|
||||
httpEnabled := !onlyGRPC
|
||||
|
||||
v3c := v3client.New(s)
|
||||
servElection := v3election.NewElectionServer(v3c)
|
||||
servLock := v3lock.NewLockServer(v3c)
|
||||
@@ -116,104 +124,137 @@ func (sctx *serveCtx) serve(
|
||||
return err
|
||||
}
|
||||
}
|
||||
var traffic string
|
||||
switch {
|
||||
case onlyGRPC:
|
||||
traffic = "grpc"
|
||||
case onlyHttp:
|
||||
traffic = "http"
|
||||
default:
|
||||
traffic = "grpc+http"
|
||||
}
|
||||
|
||||
if sctx.insecure {
|
||||
gs := v3rpc.Server(s, nil, nil, gopts...)
|
||||
v3electionpb.RegisterElectionServer(gs, servElection)
|
||||
v3lockpb.RegisterLockServer(gs, servLock)
|
||||
if sctx.serviceRegister != nil {
|
||||
sctx.serviceRegister(gs)
|
||||
}
|
||||
|
||||
defer func(gs *grpc.Server) {
|
||||
if err != nil {
|
||||
sctx.lg.Warn("stopping insecure grpc server due to error", zap.Error(err))
|
||||
gs.Stop()
|
||||
sctx.lg.Warn("stopped insecure grpc server due to error", zap.Error(err))
|
||||
var gs *grpc.Server
|
||||
var srv *http.Server
|
||||
if httpEnabled {
|
||||
httpmux := sctx.createMux(gwmux, handler)
|
||||
srv = &http.Server{
|
||||
Handler: createAccessController(sctx.lg, s, httpmux),
|
||||
ErrorLog: logger, // do not log user error
|
||||
}
|
||||
if err := configureHttpServer(srv, s.Cfg); err != nil {
|
||||
sctx.lg.Error("Configure http server failed", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
}(gs)
|
||||
|
||||
grpcl := m.Match(cmux.HTTP2())
|
||||
go func(gs *grpc.Server, grpcLis net.Listener) {
|
||||
errHandler(gs.Serve(grpcLis))
|
||||
}(gs, grpcl)
|
||||
|
||||
httpmux := sctx.createMux(gwmux, handler)
|
||||
|
||||
srvhttp := &http.Server{
|
||||
Handler: createAccessController(sctx.lg, s, httpmux),
|
||||
ErrorLog: logger, // do not log user error
|
||||
}
|
||||
if err := configureHttpServer(srvhttp, s.Cfg); err != nil {
|
||||
sctx.lg.Error("Configure http server failed", zap.Error(err))
|
||||
return err
|
||||
if grpcEnabled {
|
||||
gs = v3rpc.Server(s, nil, nil, gopts...)
|
||||
v3electionpb.RegisterElectionServer(gs, servElection)
|
||||
v3lockpb.RegisterLockServer(gs, servLock)
|
||||
if sctx.serviceRegister != nil {
|
||||
sctx.serviceRegister(gs)
|
||||
}
|
||||
defer func(gs *grpc.Server) {
|
||||
if err != nil {
|
||||
sctx.lg.Warn("stopping insecure grpc server due to error", zap.Error(err))
|
||||
gs.Stop()
|
||||
sctx.lg.Warn("stopped insecure grpc server due to error", zap.Error(err))
|
||||
}
|
||||
}(gs)
|
||||
}
|
||||
httpl := m.Match(cmux.HTTP1())
|
||||
if onlyGRPC {
|
||||
server = func() error {
|
||||
return gs.Serve(sctx.l)
|
||||
}
|
||||
} else {
|
||||
server = m.Serve
|
||||
|
||||
go func(srvhttp *http.Server, httpLis net.Listener) {
|
||||
errHandler(srvhttp.Serve(httpLis))
|
||||
}(srvhttp, httpl)
|
||||
httpl := m.Match(cmux.HTTP1())
|
||||
go func(srvhttp *http.Server, tlsLis net.Listener) {
|
||||
errHandler(srvhttp.Serve(tlsLis))
|
||||
}(srv, httpl)
|
||||
|
||||
sctx.serversC <- &servers{grpc: gs, http: srvhttp}
|
||||
if grpcEnabled {
|
||||
grpcl := m.Match(cmux.HTTP2())
|
||||
go func(gs *grpc.Server, l net.Listener) {
|
||||
errHandler(gs.Serve(l))
|
||||
}(gs, grpcl)
|
||||
}
|
||||
}
|
||||
|
||||
sctx.serversC <- &servers{grpc: gs, http: srv}
|
||||
sctx.lg.Info(
|
||||
"serving client traffic insecurely; this is strongly discouraged!",
|
||||
zap.String("traffic", traffic),
|
||||
zap.String("address", sctx.l.Addr().String()),
|
||||
)
|
||||
}
|
||||
|
||||
if sctx.secure {
|
||||
var gs *grpc.Server
|
||||
var srv *http.Server
|
||||
|
||||
tlscfg, tlsErr := tlsinfo.ServerConfig()
|
||||
if tlsErr != nil {
|
||||
return tlsErr
|
||||
}
|
||||
|
||||
gs := v3rpc.Server(s, tlscfg, nil, gopts...)
|
||||
v3electionpb.RegisterElectionServer(gs, servElection)
|
||||
v3lockpb.RegisterLockServer(gs, servLock)
|
||||
if sctx.serviceRegister != nil {
|
||||
sctx.serviceRegister(gs)
|
||||
}
|
||||
|
||||
defer func(gs *grpc.Server) {
|
||||
if err != nil {
|
||||
sctx.lg.Warn("stopping secure grpc server due to error", zap.Error(err))
|
||||
gs.Stop()
|
||||
sctx.lg.Warn("stopped secure grpc server due to error", zap.Error(err))
|
||||
if grpcEnabled {
|
||||
gs = v3rpc.Server(s, tlscfg, nil, gopts...)
|
||||
v3electionpb.RegisterElectionServer(gs, servElection)
|
||||
v3lockpb.RegisterLockServer(gs, servLock)
|
||||
if sctx.serviceRegister != nil {
|
||||
sctx.serviceRegister(gs)
|
||||
}
|
||||
}(gs)
|
||||
|
||||
handler = grpcHandlerFunc(gs, handler)
|
||||
var tlsl net.Listener
|
||||
tlsl, err = transport.NewTLSListener(m.Match(cmux.Any()), tlsinfo)
|
||||
if err != nil {
|
||||
return err
|
||||
defer func(gs *grpc.Server) {
|
||||
if err != nil {
|
||||
sctx.lg.Warn("stopping secure grpc server due to error", zap.Error(err))
|
||||
gs.Stop()
|
||||
sctx.lg.Warn("stopped secure grpc server due to error", zap.Error(err))
|
||||
}
|
||||
}(gs)
|
||||
}
|
||||
// TODO: add debug flag; enable logging when debug flag is set
|
||||
httpmux := sctx.createMux(gwmux, handler)
|
||||
if httpEnabled {
|
||||
if grpcEnabled {
|
||||
handler = grpcHandlerFunc(gs, handler)
|
||||
}
|
||||
httpmux := sctx.createMux(gwmux, handler)
|
||||
|
||||
srv := &http.Server{
|
||||
Handler: createAccessController(sctx.lg, s, httpmux),
|
||||
TLSConfig: tlscfg,
|
||||
ErrorLog: logger, // do not log user error
|
||||
}
|
||||
if err := configureHttpServer(srv, s.Cfg); err != nil {
|
||||
sctx.lg.Error("Configure https server failed", zap.Error(err))
|
||||
return err
|
||||
srv = &http.Server{
|
||||
Handler: createAccessController(sctx.lg, s, httpmux),
|
||||
TLSConfig: tlscfg,
|
||||
ErrorLog: logger, // do not log user error
|
||||
}
|
||||
if err := configureHttpServer(srv, s.Cfg); err != nil {
|
||||
sctx.lg.Error("Configure https server failed", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
go func(srvhttp *http.Server, tlsLis net.Listener) {
|
||||
errHandler(srvhttp.Serve(tlsLis))
|
||||
}(srv, tlsl)
|
||||
if onlyGRPC {
|
||||
server = func() error { return gs.Serve(sctx.l) }
|
||||
} else {
|
||||
server = m.Serve
|
||||
|
||||
tlsl, err := transport.NewTLSListener(m.Match(cmux.Any()), tlsinfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func(srvhttp *http.Server, tlsl net.Listener) {
|
||||
errHandler(srvhttp.Serve(tlsl))
|
||||
}(srv, tlsl)
|
||||
}
|
||||
|
||||
sctx.serversC <- &servers{secure: true, grpc: gs, http: srv}
|
||||
sctx.lg.Info(
|
||||
"serving client traffic securely",
|
||||
zap.String("traffic", traffic),
|
||||
zap.String("address", sctx.l.Addr().String()),
|
||||
)
|
||||
}
|
||||
|
||||
close(sctx.serversC)
|
||||
return m.Serve()
|
||||
return server()
|
||||
}
|
||||
|
||||
func configureHttpServer(srv *http.Server, cfg config.ServerConfig) error {
|
||||
|
||||
Reference in New Issue
Block a user