Merge pull request #12774 from hexfusion/add-log-rotate

server: add support for log rotation

server/embed/config.go

@@ -68,6 +68,15 @@ const (
 	StdErrLogOutput  = "stderr"
 	StdOutLogOutput  = "stdout"
 
+	// DefaultLogRotationConfig is the default configuration used for log rotation.
+	// Log rotation is disabled by default.
+	// MaxSize    = 100 // MB
+	// MaxAge     = 0 // days (no limit)
+	// MaxBackups = 0 // no limit
+	// LocalTime  = false // use computers local time, UTC by default
+	// Compress   = false // compress the rotated log in gzip format
+	DefaultLogRotationConfig = `{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}`
+
 	// DefaultStrictReconfigCheck is the default value for "--strict-reconfig-check" flag.
 	// It's enabled by default.
 	DefaultStrictReconfigCheck = true
@@ -86,6 +95,7 @@ var (
 	ErrConflictBootstrapFlags = fmt.Errorf("multiple discovery or bootstrap flags are set. " +
 		"Choose one of \"initial-cluster\", \"discovery\" or \"discovery-srv\"")
 	ErrUnsetAdvertiseClientURLsFlag = fmt.Errorf("--advertise-client-urls is required when --listen-client-urls is set explicitly")
+	ErrLogRotationInvalidLogOutput  = fmt.Errorf("--log-outputs requires a single file path when --log-rotate-config-json is defined")
 
 	DefaultInitialAdvertisePeerURLs = "http://localhost:2380"
 	DefaultAdvertiseClientURLs      = "http://localhost:2379"
@@ -320,7 +330,10 @@ type Config struct {
 	//  - file path to append server logs to.
 	// It can be multiple when "Logger" is zap.
 	LogOutputs []string `json:"log-outputs"`
-
+	// EnableLogRotation enables log rotation of a single LogOutputs file target.
+	EnableLogRotation bool `json:"enable-log-rotation"`
+	// LogRotationConfigJSON is a passthrough allowing a log rotation JSON config to be passed directly.
+	LogRotationConfigJSON string `json:"log-rotation-config-json"`
 	// ZapLoggerBuilder is used to build the zap logger.
 	ZapLoggerBuilder func(*Config) error
 
@@ -440,12 +453,14 @@ func NewConfig() *Config {
 
 		PreVote: true,
 
-		loggerMu:          new(sync.RWMutex),
-		logger:            nil,
-		Logger:            "zap",
-		LogOutputs:        []string{DefaultLogOutput},
-		LogLevel:          logutil.DefaultLogLevel,
-		EnableGRPCGateway: true,
+		loggerMu:              new(sync.RWMutex),
+		logger:                nil,
+		Logger:                "zap",
+		LogOutputs:            []string{DefaultLogOutput},
+		LogLevel:              logutil.DefaultLogLevel,
+		EnableLogRotation:     false,
+		LogRotationConfigJSON: DefaultLogRotationConfig,
+		EnableGRPCGateway:     true,
 
 		ExperimentalDowngradeCheckTime:           DefaultDowngradeCheckTime,
 		ExperimentalMemoryMlock:                  false,

server/embed/config_logging.go

@@ -16,8 +16,11 @@ package embed
 
 import (
 	"crypto/tls"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"os"
 
 	"go.etcd.io/etcd/client/pkg/v3/logutil"
@@ -26,6 +29,7 @@ import (
 	"go.uber.org/zap/zapgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/grpclog"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
 // GetLogger returns the logger.
@@ -54,6 +58,11 @@ func (cfg *Config) setupLogging() error {
 				}
 			}
 		}
+		if cfg.EnableLogRotation {
+			if err := setupLogRotation(cfg.LogOutputs, cfg.LogRotationConfigJSON); err != nil {
+				return err
+			}
+		}
 
 		outputPaths, errOutputPaths := make([]string, 0), make([]string, 0)
 		isJournal := false
@@ -75,8 +84,15 @@ func (cfg *Config) setupLogging() error {
 				errOutputPaths = append(errOutputPaths, StdOutLogOutput)
 
 			default:
-				outputPaths = append(outputPaths, v)
-				errOutputPaths = append(errOutputPaths, v)
+				var path string
+				if cfg.EnableLogRotation {
+					// append rotate scheme to logs managed by lumberjack log rotation
+					path = fmt.Sprintf("rotate:%s", v)
+				} else {
+					path = v
+				}
+				outputPaths = append(outputPaths, path)
+				errOutputPaths = append(errOutputPaths, path)
 			}
 		}
 
@@ -211,3 +227,48 @@ func (cfg *Config) SetupGlobalLoggers() {
 		zap.ReplaceGlobals(lg)
 	}
 }
+
+type logRotationConfig struct {
+	*lumberjack.Logger
+}
+
+// Sync implements zap.Sink
+func (logRotationConfig) Sync() error { return nil }
+
+// setupLogRotation initializes log rotation for a single file path target.
+func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error {
+	var logRotationConfig logRotationConfig
+	outputFilePaths := 0
+	for _, v := range logOutputs {
+		switch v {
+		case DefaultLogOutput, StdErrLogOutput, StdOutLogOutput:
+			continue
+		default:
+			outputFilePaths++
+		}
+	}
+	// log rotation requires file target
+	if len(logOutputs) == 1 && outputFilePaths == 0 {
+		return ErrLogRotationInvalidLogOutput
+	}
+	// support max 1 file target for log rotation
+	if outputFilePaths > 1 {
+		return ErrLogRotationInvalidLogOutput
+	}
+
+	if err := json.Unmarshal([]byte(logRotateConfigJSON), &logRotationConfig); err != nil {
+		var unmarshalTypeError *json.UnmarshalTypeError
+		var syntaxError *json.SyntaxError
+		switch {
+		case errors.As(err, &syntaxError):
+			return fmt.Errorf("improperly formatted log rotation config: %w", err)
+		case errors.As(err, &unmarshalTypeError):
+			return fmt.Errorf("invalid log rotation config: %w", err)
+		}
+	}
+	zap.RegisterSink("rotate", func(u *url.URL) (zap.Sink, error) {
+		logRotationConfig.Filename = u.Path
+		return &logRotationConfig, nil
+	})
+	return nil
+}

server/embed/config_test.go

@@ -15,6 +15,7 @@
 package embed
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -289,3 +290,77 @@ func TestPeerURLsMapAndTokenFromSRV(t *testing.T) {
 		}
 	}
 }
+
+func TestLogRotation(t *testing.T) {
+	tests := []struct {
+		name              string
+		logOutputs        []string
+		logRotationConfig string
+		wantErr           bool
+		wantErrMsg        error
+	}{
+		{
+			name:              "mixed log output targets",
+			logOutputs:        []string{"stderr", "/tmp/path"},
+			logRotationConfig: `{"maxsize": 1}`,
+		},
+		{
+			name:              "no file targets",
+			logOutputs:        []string{"stderr"},
+			logRotationConfig: `{"maxsize": 1}`,
+			wantErr:           true,
+			wantErrMsg:        ErrLogRotationInvalidLogOutput,
+		},
+		{
+			name:              "multiple file targets",
+			logOutputs:        []string{"/tmp/path1", "/tmp/path2"},
+			logRotationConfig: DefaultLogRotationConfig,
+			wantErr:           true,
+			wantErrMsg:        ErrLogRotationInvalidLogOutput,
+		},
+		{
+			name:              "default output",
+			logRotationConfig: `{"maxsize": 1}`,
+			wantErr:           true,
+			wantErrMsg:        ErrLogRotationInvalidLogOutput,
+		},
+		{
+			name:              "default log rotation config",
+			logOutputs:        []string{"/tmp/path"},
+			logRotationConfig: DefaultLogRotationConfig,
+		},
+		{
+			name:              "invalid logger config",
+			logOutputs:        []string{"/tmp/path"},
+			logRotationConfig: `{"maxsize": true}`,
+			wantErr:           true,
+			wantErrMsg:        errors.New("invalid log rotation config: json: cannot unmarshal bool into Go struct field logRotationConfig.maxsize of type int"),
+		},
+		{
+			name:              "improperly formatted logger config",
+			logOutputs:        []string{"/tmp/path"},
+			logRotationConfig: `{"maxsize": true`,
+			wantErr:           true,
+			wantErrMsg:        errors.New("improperly formatted log rotation config: unexpected end of JSON input"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := NewConfig()
+			cfg.Logger = "zap"
+			cfg.LogOutputs = tt.logOutputs
+			cfg.EnableLogRotation = true
+			cfg.LogRotationConfigJSON = tt.logRotationConfig
+			err := cfg.Validate()
+			if err != nil && !tt.wantErr {
+				t.Errorf("test %q, unexpected error %v", tt.name, err)
+			}
+			if err != nil && tt.wantErr && tt.wantErrMsg.Error() != err.Error() {
+				t.Errorf("test %q, expected error: %+v, got: %+v", tt.name, tt.wantErrMsg, err)
+			}
+			if err == nil && tt.wantErr {
+				t.Errorf("test %q, expected error, got nil", tt.name)
+			}
+		})
+	}
+}