From a46a358577c38fef89740f4727a99a1ca9eb3f1b Mon Sep 17 00:00:00 2001 From: Piotr Tabor Date: Fri, 5 Mar 2021 16:17:59 +0100 Subject: [PATCH] --experimental-memory-mlock support The flag protects etcd memory from being swapped out to disk. This can happen in memory constrained systems where mmaped bbolt area is natural condidate for swapping out. This flag should provide better tail latency on the cost of higher RSS ram usage. If the experiment is successful, the logic should get moved into bbolt layer, where we can protect specific bbolt instances (e.g. avoid protecting both during defragmentation). --- CHANGELOG-3.5.md | 2 +- client/v2/go.sum | 2 +- client/v3/go.sum | 4 ++-- etcdctl/go.sum | 4 ++-- go.sum | 4 ++-- pkg/go.mod | 2 +- pkg/go.sum | 4 ++-- raft/go.sum | 2 +- server/embed/config.go | 9 ++++++++ server/embed/etcd.go | 1 + server/etcdmain/config.go | 1 + server/etcdserver/config.go | 8 +++++++ server/etcdserver/mlock_not_unix.go | 27 +++++++++++++++++++++++ server/etcdserver/mlock_unix.go | 33 +++++++++++++++++++++++++++++ server/etcdserver/server.go | 8 +++++++ server/go.mod | 1 + server/go.sum | 4 ++-- tests/go.sum | 4 ++-- 18 files changed, 104 insertions(+), 16 deletions(-) create mode 100644 server/etcdserver/mlock_not_unix.go create mode 100644 server/etcdserver/mlock_unix.go diff --git a/CHANGELOG-3.5.md b/CHANGELOG-3.5.md index 918079652..f5b1bfcf1 100644 --- a/CHANGELOG-3.5.md +++ b/CHANGELOG-3.5.md @@ -149,7 +149,7 @@ Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`--self-signed-cert-validity`](https://github.com/etcd-io/etcd/pull/12429) flag to support setting certificate expiration time. - Notice, certificates generated by etcd are valid for 1 year by default when specifying the auto-tls or peer-auto-tls option. - Add [`--experimental-warning-apply-duration`](https://github.com/etcd-io/etcd/pull/12448) flag which allows apply duration threshold to be configurable. - +- Add [`--experimental-memory-mlock`](https://github.com/etcd-io/etcd/pull/TODO)" flag which prevents etcd memory pages to be swapped out. ### Package `runtime` - Optimize [`runtime.FDUsage` by removing unnecessary sorting](https://github.com/etcd-io/etcd/pull/12214). diff --git a/client/v2/go.sum b/client/v2/go.sum index 3d222cdaf..94d0db7e4 100644 --- a/client/v2/go.sum +++ b/client/v2/go.sum @@ -85,7 +85,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/client/v3/go.sum b/client/v3/go.sum index 12d5814bc..bb11df3c8 100644 --- a/client/v3/go.sum +++ b/client/v3/go.sum @@ -151,8 +151,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0 h1:MOJR6AyRlIYMexU2acorBot1aPks0cBDOyUA4hFlBhE= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/etcdctl/go.sum b/etcdctl/go.sum index 5efcfe267..ca775b943 100644 --- a/etcdctl/go.sum +++ b/etcdctl/go.sum @@ -374,8 +374,8 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0 h1:MOJR6AyRlIYMexU2acorBot1aPks0cBDOyUA4hFlBhE= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/go.sum b/go.sum index de9b894fd..a66983d14 100644 --- a/go.sum +++ b/go.sum @@ -378,8 +378,8 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0 h1:MOJR6AyRlIYMexU2acorBot1aPks0cBDOyUA4hFlBhE= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/pkg/go.mod b/pkg/go.mod index 6c4615514..4f6d8e4d2 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -8,7 +8,7 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/spf13/pflag v1.0.5 go.uber.org/zap v1.16.0 - golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 + golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0 google.golang.org/grpc v1.32.0 ) diff --git a/pkg/go.sum b/pkg/go.sum index 54b0fe396..4ed1d8491 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -76,8 +76,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0 h1:MOJR6AyRlIYMexU2acorBot1aPks0cBDOyUA4hFlBhE= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/raft/go.sum b/raft/go.sum index c69a4a613..4cc976779 100644 --- a/raft/go.sum +++ b/raft/go.sum @@ -78,7 +78,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/server/embed/config.go b/server/embed/config.go index be22efa5c..3fd3c2d6a 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -345,6 +345,14 @@ type Config struct { UnsafeNoFsync bool `json:"unsafe-no-fsync"` ExperimentalDowngradeCheckTime time.Duration `json:"experimental-downgrade-check-time"` + + // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // The setting improves etcd tail latency in environments were: + // - memory pressure might lead to swapping pages to disk + // - disk latency might be unstable + // Currently all etcd memory gets mlocked, but in future the flag can + // be refined to mlock in-use area of bbolt only. + ExperimentalMemoryMlock bool `json:"experimental-memory-mlock"` } // configYAML holds the config suitable for yaml parsing @@ -433,6 +441,7 @@ func NewConfig() *Config { EnableGRPCGateway: true, ExperimentalDowngradeCheckTime: DefaultDowngradeCheckTime, + ExperimentalMemoryMlock: false, } cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) return cfg diff --git a/server/embed/etcd.go b/server/embed/etcd.go index c5b99a9ff..b2cd81250 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -203,6 +203,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { WatchProgressNotifyInterval: cfg.ExperimentalWatchProgressNotifyInterval, DowngradeCheckTime: cfg.ExperimentalDowngradeCheckTime, WarningApplyDuration: cfg.ExperimentalWarningApplyDuration, + ExperimentalMemoryMlock: cfg.ExperimentalMemoryMlock, } print(e.cfg.logger, *cfg, srvcfg, memberInitialized) if e.Server, err = etcdserver.NewServer(srvcfg); err != nil { diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index ef00427e9..d7cb31dfd 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -264,6 +264,7 @@ func newConfig() *config { fs.DurationVar(&cfg.ec.ExperimentalWatchProgressNotifyInterval, "experimental-watch-progress-notify-interval", cfg.ec.ExperimentalWatchProgressNotifyInterval, "Duration of periodic watch progress notifications.") fs.DurationVar(&cfg.ec.ExperimentalDowngradeCheckTime, "experimental-downgrade-check-time", cfg.ec.ExperimentalDowngradeCheckTime, "Duration of time between two downgrade status check.") fs.DurationVar(&cfg.ec.ExperimentalWarningApplyDuration, "experimental-warning-apply-duration", cfg.ec.ExperimentalWarningApplyDuration, "Time duration after which a warning is generated if request takes more time.") + fs.BoolVar(&cfg.ec.ExperimentalMemoryMlock, "experimental-memory-mlock", cfg.ec.ExperimentalMemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") // unsafe fs.BoolVar(&cfg.ec.UnsafeNoFsync, "unsafe-no-fsync", false, "Disables fsync, unsafe, will cause data loss.") diff --git a/server/etcdserver/config.go b/server/etcdserver/config.go index 49fe04005..50cfdfd8d 100644 --- a/server/etcdserver/config.go +++ b/server/etcdserver/config.go @@ -166,6 +166,14 @@ type ServerConfig struct { UnsafeNoFsync bool `json:"unsafe-no-fsync"` DowngradeCheckTime time.Duration + + // ExperimentalMemoryMlock enables mlocking of etcd owned memory pages. + // The setting improves etcd tail latency in environments were: + // - memory pressure might lead to swapping pages to disk + // - disk latency might be unstable + // Currently all etcd memory gets mlocked, but in future the flag can + // be refined to mlock in-use area of bbolt only. + ExperimentalMemoryMlock bool `json:"experimental-memory-mlock"` } // VerifyBootstrap sanity-checks the initial config for bootstrap case diff --git a/server/etcdserver/mlock_not_unix.go b/server/etcdserver/mlock_not_unix.go new file mode 100644 index 000000000..6758819a7 --- /dev/null +++ b/server/etcdserver/mlock_not_unix.go @@ -0,0 +1,27 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows +// +build windows + +package etcdserver + +import ( + "errors" +) + +// MlockAll prevents current and future mmaped memory areas from being swapped out. +func MlockAll() error { + return errors.New("Mlockall is supported only on UNIX systems.") +} diff --git a/server/etcdserver/mlock_unix.go b/server/etcdserver/mlock_unix.go new file mode 100644 index 000000000..5e35fad0b --- /dev/null +++ b/server/etcdserver/mlock_unix.go @@ -0,0 +1,33 @@ +// Copyright 2021 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris || zos +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris zos + +package etcdserver + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +// MlockAll prevents current and future mmaped memory areas from being swapped out. +func MlockAll() error { + err := unix.Mlockall(unix.MCL_FUTURE | unix.MCL_CURRENT) + if err != nil { + return fmt.Errorf("cannot mlockAll: %v", err) + } + return nil +} diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index 01fdc0c3f..662458a6d 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -316,6 +316,14 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) { ) } + if cfg.ExperimentalMemoryMlock { + cfg.Logger.Info("mlocking memory") + err := MlockAll() + if err != nil { + return nil, err + } + } + if terr := fileutil.TouchDirAll(cfg.DataDir); terr != nil { return nil, fmt.Errorf("cannot access data directory: %v", terr) } diff --git a/server/go.mod b/server/go.mod index 006c15211..d2a20809b 100644 --- a/server/go.mod +++ b/server/go.mod @@ -37,6 +37,7 @@ require ( golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb + golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e golang.org/x/tools v0.0.0-20201014170642-d1624618ad65 // indirect google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884 diff --git a/server/go.sum b/server/go.sum index bbc67e42b..324fd938d 100644 --- a/server/go.sum +++ b/server/go.sum @@ -367,8 +367,8 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0 h1:MOJR6AyRlIYMexU2acorBot1aPks0cBDOyUA4hFlBhE= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/tests/go.sum b/tests/go.sum index 82f0aefd5..02ef9405c 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -374,8 +374,8 @@ golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0 h1:MOJR6AyRlIYMexU2acorBot1aPks0cBDOyUA4hFlBhE= +golang.org/x/sys v0.0.0-20210305215415-5cdee2b1b5a0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=