etcd/server/embed/config_logging.go
Piotr Tabor 8ccd4e1146 Fix flaky tests reported due to data race on grpc logging registration.
Example:
```
    ==================
    WARNING: DATA RACE
    Write at 0x000002178320 by goroutine 575:
      google.golang.org/grpc/grpclog.SetLoggerV2()
          /home/ptab/private/golang/pkg/mod/google.golang.org/grpc@v1.29.1/grpclog/loggerv2.go:70 +0x444
      go.etcd.io/etcd/server/v3/embed.(*Config).setupLogging.func1.1()
          /home/ptab/corp/etcd/server/embed/config_logging.go:119 +0x345
      sync.(*Once).doSlow()
          /usr/lib/google-golang/src/sync/once.go:66 +0x109
      sync.(*Once).Do()
          /usr/lib/google-golang/src/sync/once.go:57 +0x68
      go.etcd.io/etcd/server/v3/embed.(*Config).setupLogging.func1()
          /home/ptab/corp/etcd/server/embed/config_logging.go:109 +0x3b1
      go.etcd.io/etcd/server/v3/embed.(*Config).setupLogging()
          /home/ptab/corp/etcd/server/embed/config_logging.go:174 +0x6af
      go.etcd.io/etcd/server/v3/embed.(*Config).Validate()
          /home/ptab/corp/etcd/server/embed/config.go:553 +0x55
      go.etcd.io/etcd/server/v3/embed.StartEtcd()
          /home/ptab/corp/etcd/server/embed/etcd.go:93 +0x84
      go.etcd.io/etcd/tests/v3/integration.TestKVWithEmptyValue()
          /home/ptab/corp/etcd/tests/integration/v3_kv_test.go:33 +0x18c
      testing.tRunner()
          /usr/lib/google-golang/src/testing/testing.go:1123 +0x202

    Previous read at 0x000002178320 by goroutine 956:
      [failed to restore the stack]

    Goroutine 575 (running) created at:
      testing.(*T).Run()
          /usr/lib/google-golang/src/testing/testing.go:1168 +0x5bb
      testing.runTests.func1()
          /usr/lib/google-golang/src/testing/testing.go:1441 +0xa6
      testing.tRunner()
          /usr/lib/google-golang/src/testing/testing.go:1123 +0x202
      testing.runTests()
          /usr/lib/google-golang/src/testing/testing.go:1439 +0x612
      testing.(*M).Run()
          /usr/lib/google-golang/src/testing/testing.go:1347 +0x3c4
      go.etcd.io/etcd/pkg/v3/testutil.MustTestMainWithLeakDetection()
          /home/ptab/corp/etcd/pkg/testutil/leak.go:150 +0x38
      go.etcd.io/etcd/tests/v3/integration.TestMain()
          /home/ptab/corp/etcd/tests/integration/main_test.go:14 +0x272
      main.main()
          _testmain.go:349 +0x269

    Goroutine 956 (finished) created at:
      google.golang.org/grpc/internal/transport.newHTTP2Server()
          /home/ptab/private/golang/pkg/mod/google.golang.org/grpc@v1.29.1/internal/transport/http2_server.go:288 +0x18a4
      google.golang.org/grpc/internal/transport.NewServerTransport()
          /home/ptab/private/golang/pkg/mod/google.golang.org/grpc@v1.29.1/internal/transport/transport.go:534 +0x2f5
      google.golang.org/grpc.(*Server).newHTTP2Transport()
          /home/ptab/private/golang/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:726 +0x2ca
      google.golang.org/grpc.(*Server).handleRawConn()
          /home/ptab/private/golang/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:693 +0x60f
      google.golang.org/grpc.(*Server).Serve.func3()
          /home/ptab/private/golang/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:663 +0x4c
    ==================
    ...
    {"level":"info","ts":"2021-01-09T22:21:04.550+0100","caller":"embed/etcd.go:330","msg":"closed etcd server","name":"default","data-dir":"/tmp/etcd-017337431","advertise-peer-urls":["http://localhost:2380"],"advertise-client-urls":["http://localhost:2379"]}
    --- FAIL: TestKVWithEmptyValue (1.08s)
        v3_kv_test.go:62: my-namespace/foobar = data
        v3_kv_test.go:62: my-namespace/foobar1 = data
        v3_kv_test.go:62: namespace/foobar1 = data
        v3_kv_test.go:72: foobar = data
        v3_kv_test.go:72: foobar1 = data
        v3_kv_test.go:87: delete keys:2
        testing.go:1038: race detected during execution of test
```
2021-01-11 10:06:31 +01:00

240 lines
6.5 KiB
Go

// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package embed
import (
"crypto/tls"
"fmt"
"io/ioutil"
"os"
"sync"
"go.etcd.io/etcd/pkg/v3/logutil"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"google.golang.org/grpc"
"google.golang.org/grpc/grpclog"
)
// GetLogger returns the logger.
func (cfg Config) GetLogger() *zap.Logger {
cfg.loggerMu.RLock()
l := cfg.logger
cfg.loggerMu.RUnlock()
return l
}
// for testing
var grpcLogOnce = new(sync.Once)
func setupGrpcLogging(debug bool, config zap.Config) {
grpcLogOnce.Do(func() {
// debug true, enable info, warning, error
// debug false, only discard info
if debug {
var gl grpclog.LoggerV2
gl, err := logutil.NewGRPCLoggerV2(config)
if err == nil {
grpclog.SetLoggerV2(gl)
}
} else {
grpclog.SetLoggerV2(grpclog.NewLoggerV2(ioutil.Discard, os.Stderr, os.Stderr))
}
})
}
func SetupGrpcLoggingForTest(debug bool) {
setupGrpcLogging(debug, zap.NewDevelopmentConfig())
}
// setupLogging initializes etcd logging.
// Must be called after flag parsing or finishing configuring embed.Config.
func (cfg *Config) setupLogging() error {
switch cfg.Logger {
case "capnslog": // removed in v3.5
return fmt.Errorf("--logger=capnslog is removed in v3.5")
case "zap":
if len(cfg.LogOutputs) == 0 {
cfg.LogOutputs = []string{DefaultLogOutput}
}
if len(cfg.LogOutputs) > 1 {
for _, v := range cfg.LogOutputs {
if v == DefaultLogOutput {
return fmt.Errorf("multi logoutput for %q is not supported yet", DefaultLogOutput)
}
}
}
outputPaths, errOutputPaths := make([]string, 0), make([]string, 0)
isJournal := false
for _, v := range cfg.LogOutputs {
switch v {
case DefaultLogOutput:
outputPaths = append(outputPaths, StdErrLogOutput)
errOutputPaths = append(errOutputPaths, StdErrLogOutput)
case JournalLogOutput:
isJournal = true
case StdErrLogOutput:
outputPaths = append(outputPaths, StdErrLogOutput)
errOutputPaths = append(errOutputPaths, StdErrLogOutput)
case StdOutLogOutput:
outputPaths = append(outputPaths, StdOutLogOutput)
errOutputPaths = append(errOutputPaths, StdOutLogOutput)
default:
outputPaths = append(outputPaths, v)
errOutputPaths = append(errOutputPaths, v)
}
}
if !isJournal {
copied := logutil.DefaultZapLoggerConfig
copied.OutputPaths = outputPaths
copied.ErrorOutputPaths = errOutputPaths
copied = logutil.MergeOutputPaths(copied)
copied.Level = zap.NewAtomicLevelAt(logutil.ConvertToZapLevel(cfg.LogLevel))
if cfg.LogLevel == "debug" {
grpc.EnableTracing = true
}
if cfg.ZapLoggerBuilder == nil {
cfg.ZapLoggerBuilder = func(c *Config) error {
var err error
c.logger, err = copied.Build()
if err != nil {
return err
}
zap.ReplaceGlobals(c.logger)
c.loggerMu.Lock()
defer c.loggerMu.Unlock()
c.loggerConfig = &copied
c.loggerCore = nil
c.loggerWriteSyncer = nil
setupGrpcLogging(cfg.LogLevel == "debug", copied)
return nil
}
}
} else {
if len(cfg.LogOutputs) > 1 {
for _, v := range cfg.LogOutputs {
if v != DefaultLogOutput {
return fmt.Errorf("running with systemd/journal but other '--log-outputs' values (%q) are configured with 'default'; override 'default' value with something else", cfg.LogOutputs)
}
}
}
// use stderr as fallback
syncer, lerr := getJournalWriteSyncer()
if lerr != nil {
return lerr
}
lvl := zap.NewAtomicLevelAt(logutil.ConvertToZapLevel(cfg.LogLevel))
if cfg.LogLevel == "debug" {
grpc.EnableTracing = true
}
// WARN: do not change field names in encoder config
// journald logging writer assumes field names of "level" and "caller"
cr := zapcore.NewCore(
zapcore.NewJSONEncoder(logutil.DefaultZapLoggerConfig.EncoderConfig),
syncer,
lvl,
)
if cfg.ZapLoggerBuilder == nil {
cfg.ZapLoggerBuilder = func(c *Config) error {
c.logger = zap.New(cr, zap.AddCaller(), zap.ErrorOutput(syncer))
zap.ReplaceGlobals(c.logger)
c.loggerMu.Lock()
defer c.loggerMu.Unlock()
c.loggerConfig = nil
c.loggerCore = cr
c.loggerWriteSyncer = syncer
grpcLogOnce.Do(func() {
if cfg.LogLevel == "debug" {
grpclog.SetLoggerV2(logutil.NewGRPCLoggerV2FromZapCore(cr, syncer))
} else {
grpclog.SetLoggerV2(grpclog.NewLoggerV2(ioutil.Discard, os.Stderr, os.Stderr))
}
})
return nil
}
}
}
err := cfg.ZapLoggerBuilder(cfg)
if err != nil {
return err
}
logTLSHandshakeFailure := func(conn *tls.Conn, err error) {
state := conn.ConnectionState()
remoteAddr := conn.RemoteAddr().String()
serverName := state.ServerName
if len(state.PeerCertificates) > 0 {
cert := state.PeerCertificates[0]
ips := make([]string, len(cert.IPAddresses))
for i := range cert.IPAddresses {
ips[i] = cert.IPAddresses[i].String()
}
cfg.logger.Warn(
"rejected connection",
zap.String("remote-addr", remoteAddr),
zap.String("server-name", serverName),
zap.Strings("ip-addresses", ips),
zap.Strings("dns-names", cert.DNSNames),
zap.Error(err),
)
} else {
cfg.logger.Warn(
"rejected connection",
zap.String("remote-addr", remoteAddr),
zap.String("server-name", serverName),
zap.Error(err),
)
}
}
cfg.ClientTLSInfo.HandshakeFailure = logTLSHandshakeFailure
cfg.PeerTLSInfo.HandshakeFailure = logTLSHandshakeFailure
default:
return fmt.Errorf("unknown logger option %q", cfg.Logger)
}
return nil
}
// NewZapCoreLoggerBuilder generates a zap core logger builder.
func NewZapCoreLoggerBuilder(lg *zap.Logger, cr zapcore.Core, syncer zapcore.WriteSyncer) func(*Config) error {
return func(cfg *Config) error {
cfg.loggerMu.Lock()
defer cfg.loggerMu.Unlock()
cfg.logger = lg
cfg.loggerConfig = nil
cfg.loggerCore = cr
cfg.loggerWriteSyncer = syncer
grpcLogOnce.Do(func() {
grpclog.SetLoggerV2(logutil.NewGRPCLoggerV2FromZapCore(cr, syncer))
})
return nil
}
}