From 370f9cf3b9f9898d2b22ea709e7441772f1039af Mon Sep 17 00:00:00 2001 From: tangcong Date: Mon, 26 Apr 2021 11:43:50 +0800 Subject: [PATCH 1/3] fix: failed to get failpoints from member --- tests/functional/tester/cluster.go | 2 -- tests/functional/tester/cluster_run.go | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/functional/tester/cluster.go b/tests/functional/tester/cluster.go index befc07579..44cff91ea 100644 --- a/tests/functional/tester/cluster.go +++ b/tests/functional/tester/cluster.go @@ -113,8 +113,6 @@ func NewCluster(lg *zap.Logger, fpath string) (*Cluster, error) { } go clus.serveTesterServer() - clus.updateCases() - clus.rateLimiter = rate.NewLimiter( rate.Limit(int(clus.Tester.StressQPS)), int(clus.Tester.StressQPS), diff --git a/tests/functional/tester/cluster_run.go b/tests/functional/tester/cluster_run.go index 8529c514e..ac65ebca3 100644 --- a/tests/functional/tester/cluster_run.go +++ b/tests/functional/tester/cluster_run.go @@ -33,6 +33,10 @@ const compactQPS = 50000 func (clus *Cluster) Run() { defer printReport() + // updateCases must be executed after etcd is started, because the FAILPOINTS case + // needs to obtain all the failpoints from the etcd member. + clus.updateCases() + if err := fileutil.TouchDirAll(clus.Tester.DataDir); err != nil { clus.lg.Panic( "failed to create test data directory", From 16e38e49a9b36e945630eead9686ef782be015b1 Mon Sep 17 00:00:00 2001 From: tangcong Date: Mon, 26 Apr 2021 12:07:05 +0800 Subject: [PATCH 2/3] functional: add FAILPOINTS_WITH_DISK_IO_LATENCY case --- tests/functional/rpcpb/rpc.proto | 3 + .../tester/case_failpoints_disk_io.go | 71 +++++++++++++++++++ tests/functional/tester/cluster.go | 7 ++ 3 files changed, 81 insertions(+) create mode 100644 tests/functional/tester/case_failpoints_disk_io.go diff --git a/tests/functional/rpcpb/rpc.proto b/tests/functional/rpcpb/rpc.proto index 79e0d352b..7d0fc6f31 100644 --- a/tests/functional/rpcpb/rpc.proto +++ b/tests/functional/rpcpb/rpc.proto @@ -625,6 +625,9 @@ enum Case { // in critical code paths. FAILPOINTS = 400; + // FAILPOINTS_WITH_DISK_IO_LATENCY injects high disk I/O latency failure in raftAfterSave code paths. + FAILPOINTS_WITH_DISK_IO_LATENCY = 401; + // EXTERNAL runs external failure injection scripts. EXTERNAL = 500; } diff --git a/tests/functional/tester/case_failpoints_disk_io.go b/tests/functional/tester/case_failpoints_disk_io.go new file mode 100644 index 000000000..4cc2396b6 --- /dev/null +++ b/tests/functional/tester/case_failpoints_disk_io.go @@ -0,0 +1,71 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "fmt" + "strings" + + "go.etcd.io/etcd/tests/v3/functional/rpcpb" +) + +const ( + diskIOFailpoint = "raftAfterSave" +) + +func failpointDiskIOFailures(clus *Cluster) (ret []Case, err error) { + fps, err := failpointPaths(clus.Members[0].FailpointHTTPAddr) + if err != nil { + return nil, err + } + var detailDiskIOLatencyFailpointPath string + for i := 0; i < len(fps); i++ { + if strings.HasSuffix(fps[i], diskIOFailpoint) { + detailDiskIOLatencyFailpointPath = fps[i] + break + } + } + // create failure objects for diskIOFailpoint + fpFails := casesFromDiskIOFailpoint(detailDiskIOLatencyFailpointPath, clus.Tester.FailpointCommands) + // wrap in delays so failpoint has time to trigger + for i, fpf := range fpFails { + fpFails[i] = &caseDelay{ + Case: fpf, + delayDuration: clus.GetCaseDelayDuration(), + } + } + ret = append(ret, fpFails...) + return ret, nil +} + +func casesFromDiskIOFailpoint(fp string, failpointCommands []string) (fs []Case) { + recov := makeRecoverFailpoint(fp) + for _, fcmd := range failpointCommands { + inject := makeInjectFailpoint(fp, fcmd) + fs = append(fs, []Case{ + &caseLeader{ + caseByFunc: caseByFunc{ + desc: fmt.Sprintf("failpoint %q (leader: %q)", fp, fcmd), + rpcpbCase: rpcpb.Case_FAILPOINTS, + injectMember: inject, + recoverMember: recov, + }, + last: -1, + lead: -1, + }, + }...) + } + return fs +} diff --git a/tests/functional/tester/cluster.go b/tests/functional/tester/cluster.go index 44cff91ea..25d478a14 100644 --- a/tests/functional/tester/cluster.go +++ b/tests/functional/tester/cluster.go @@ -259,6 +259,13 @@ func (clus *Cluster) updateCases() { } clus.cases = append(clus.cases, fpFailures...) + case "FAILPOINTS_WITH_DISK_IO_LATENCY": + fpFailures, fperr := failpointDiskIOFailures(clus) + if len(fpFailures) == 0 { + clus.lg.Info("no failpoints found!", zap.Error(fperr)) + } + clus.cases = append(clus.cases, + fpFailures...) } } } From 4fb22093a6d1eddeba38ea5465756f265e2ab408 Mon Sep 17 00:00:00 2001 From: tangcong Date: Mon, 26 Apr 2021 14:02:37 +0800 Subject: [PATCH 3/3] functional: add SHORT_TTL_LEASE_EXPIRE checker --- tests/functional/functional.yaml | 3 + tests/functional/rpcpb/rpc.pb.go | 436 +++++++++--------- tests/functional/rpcpb/rpc.proto | 1 + .../functional/tester/checker_lease_expire.go | 47 +- .../tester/checker_short_ttl_lease_expire.go | 77 ++++ tests/functional/tester/cluster.go | 5 + tests/functional/tester/stresser_lease.go | 48 +- 7 files changed, 363 insertions(+), 254 deletions(-) create mode 100644 tests/functional/tester/checker_short_ttl_lease_expire.go diff --git a/tests/functional/functional.yaml b/tests/functional/functional.yaml index 7ae03ca79..91a8e5ecd 100644 --- a/tests/functional/functional.yaml +++ b/tests/functional/functional.yaml @@ -193,6 +193,7 @@ tester-config: - RANDOM_DELAY_PEER_PORT_TX_RX_ALL - NO_FAIL_WITH_STRESS - NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS + # - FAILPOINTS_WITH_DISK_IO_LATENCY # TODO: use iptables for discarding outbound rafthttp traffic to peer port # - BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER @@ -208,6 +209,7 @@ tester-config: failpoint-commands: - panic("etcd-tester") # - panic("etcd-tester"),1*sleep(1000) + # - sleep(3000) runner-exec-path: ./bin/etcd-runner external-exec-path: "" @@ -239,6 +241,7 @@ tester-config: checkers: - KV_HASH - LEASE_EXPIRE + #- SHORT_TTL_LEASE_EXPIRE stress-key-size: 100 stress-key-size-large: 32769 diff --git a/tests/functional/rpcpb/rpc.pb.go b/tests/functional/rpcpb/rpc.pb.go index 7f6143ff6..34c953558 100644 --- a/tests/functional/rpcpb/rpc.pb.go +++ b/tests/functional/rpcpb/rpc.pb.go @@ -86,10 +86,11 @@ func (StresserType) EnumDescriptor() ([]byte, []int) { type Checker int32 const ( - Checker_KV_HASH Checker = 0 - Checker_LEASE_EXPIRE Checker = 1 - Checker_RUNNER Checker = 2 - Checker_NO_CHECK Checker = 3 + Checker_KV_HASH Checker = 0 + Checker_LEASE_EXPIRE Checker = 1 + Checker_RUNNER Checker = 2 + Checker_NO_CHECK Checker = 3 + Checker_SHORT_TTL_LEASE_EXPIRE Checker = 4 ) var Checker_name = map[int32]string{ @@ -97,13 +98,15 @@ var Checker_name = map[int32]string{ 1: "LEASE_EXPIRE", 2: "RUNNER", 3: "NO_CHECK", + 4: "SHORT_TTL_LEASE_EXPIRE", } var Checker_value = map[string]int32{ - "KV_HASH": 0, - "LEASE_EXPIRE": 1, - "RUNNER": 2, - "NO_CHECK": 3, + "KV_HASH": 0, + "LEASE_EXPIRE": 1, + "RUNNER": 2, + "NO_CHECK": 3, + "SHORT_TTL_LEASE_EXPIRE": 4, } func (x Checker) String() string { @@ -507,6 +510,8 @@ const ( // FAILPOINTS injects failpoints to etcd server runtime, triggering panics // in critical code paths. Case_FAILPOINTS Case = 400 + // FAILPOINTS_WITH_DISK_IO_LATENCY injects high disk I/O latency failure in raftAfterSave code paths. + Case_FAILPOINTS_WITH_DISK_IO_LATENCY Case = 401 // EXTERNAL runs external failure injection scripts. Case_EXTERNAL Case = 500 ) @@ -544,6 +549,7 @@ var Case_name = map[int32]string{ 300: "NO_FAIL_WITH_STRESS", 301: "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS", 400: "FAILPOINTS", + 401: "FAILPOINTS_WITH_DISK_IO_LATENCY", 500: "EXTERNAL", } @@ -580,6 +586,7 @@ var Case_value = map[string]int32{ "NO_FAIL_WITH_STRESS": 300, "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": 301, "FAILPOINTS": 400, + "FAILPOINTS_WITH_DISK_IO_LATENCY": 401, "EXTERNAL": 500, } @@ -1040,195 +1047,197 @@ func init() { func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptor_4fbc93a8dcc3881e) } var fileDescriptor_4fbc93a8dcc3881e = []byte{ - // 3003 bytes of a gzipped FileDescriptorProto + // 3039 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x59, 0xcb, 0x77, 0xdb, 0xc6, - 0xf5, 0x36, 0x44, 0x49, 0x96, 0x46, 0x2f, 0x6a, 0x64, 0xd9, 0xf0, 0x4b, 0x90, 0xe1, 0x38, 0x3f, - 0x59, 0x09, 0x6c, 0xff, 0xec, 0x9c, 0x3c, 0x9c, 0x26, 0x0e, 0x48, 0x41, 0x12, 0x2b, 0x88, 0xa4, - 0x87, 0x90, 0xec, 0x74, 0x83, 0x03, 0x91, 0x23, 0x09, 0xc7, 0x14, 0xc0, 0x00, 0x43, 0x47, 0xca, - 0x3f, 0xd0, 0x5d, 0x4f, 0xdf, 0xa7, 0x3d, 0xa7, 0x7f, 0x42, 0xd3, 0x2e, 0xba, 0xee, 0xde, 0x79, - 0xb5, 0x69, 0xbb, 0x6a, 0x17, 0x3c, 0x6d, 0xba, 0xe9, 0xaa, 0x0b, 0x9e, 0xbe, 0x17, 0x3d, 0x3d, - 0x33, 0x03, 0x90, 0x03, 0x80, 0x94, 0xbd, 0xb2, 0x70, 0xef, 0xf7, 0x7d, 0x73, 0x71, 0xef, 0x60, - 0xee, 0x1d, 0x13, 0xcc, 0x05, 0xad, 0x7a, 0x6b, 0xef, 0x76, 0xd0, 0xaa, 0xdf, 0x6a, 0x05, 0x3e, - 0xf1, 0xe1, 0x18, 0x33, 0x5c, 0xd2, 0x0e, 0x5c, 0x72, 0xd8, 0xde, 0xbb, 0x55, 0xf7, 0x8f, 0x6e, - 0x1f, 0xf8, 0x07, 0xfe, 0x6d, 0xe6, 0xdd, 0x6b, 0xef, 0xb3, 0x27, 0xf6, 0xc0, 0xfe, 0xe2, 0x2c, - 0xf5, 0x9b, 0x12, 0x38, 0x8b, 0xf0, 0x07, 0x6d, 0x1c, 0x12, 0x78, 0x0b, 0x4c, 0x56, 0x5a, 0x38, - 0x70, 0x88, 0xeb, 0x7b, 0xb2, 0xb4, 0x2c, 0xad, 0xcc, 0xde, 0xcd, 0xdf, 0x62, 0xaa, 0xb7, 0x7a, - 0x76, 0xd4, 0x87, 0xc0, 0x1b, 0x60, 0x7c, 0x1b, 0x1f, 0xed, 0xe1, 0x40, 0x1e, 0x59, 0x96, 0x56, - 0xa6, 0xee, 0xce, 0x44, 0x60, 0x6e, 0x44, 0x91, 0x93, 0xc2, 0x2c, 0x1c, 0x12, 0x1c, 0xc8, 0xb9, - 0x04, 0x8c, 0x1b, 0x51, 0xe4, 0x54, 0xff, 0x32, 0x02, 0xa6, 0x6b, 0x9e, 0xd3, 0x0a, 0x0f, 0x7d, - 0x52, 0xf2, 0xf6, 0x7d, 0xb8, 0x04, 0x00, 0x57, 0x28, 0x3b, 0x47, 0x98, 0xc5, 0x33, 0x89, 0x04, - 0x0b, 0x5c, 0x05, 0x79, 0xfe, 0x54, 0x6c, 0xba, 0xd8, 0x23, 0x3b, 0xc8, 0x0c, 0xe5, 0x91, 0xe5, - 0xdc, 0xca, 0x24, 0xca, 0xd8, 0xa1, 0xda, 0xd7, 0xae, 0x3a, 0xe4, 0x90, 0x45, 0x32, 0x89, 0x12, - 0x36, 0xaa, 0x17, 0x3f, 0xaf, 0xbb, 0x4d, 0x5c, 0x73, 0x3f, 0xc2, 0xf2, 0x28, 0xc3, 0x65, 0xec, - 0xf0, 0x55, 0x30, 0x1f, 0xdb, 0x2c, 0x9f, 0x38, 0x4d, 0x06, 0x1e, 0x63, 0xe0, 0xac, 0x43, 0x54, - 0x66, 0xc6, 0x2d, 0x7c, 0x22, 0x8f, 0x2f, 0x4b, 0x2b, 0x39, 0x94, 0xb1, 0x8b, 0x91, 0x6e, 0x3a, - 0xe1, 0xa1, 0x7c, 0x96, 0xe1, 0x12, 0x36, 0x51, 0x0f, 0xe1, 0xa7, 0x6e, 0x48, 0xeb, 0x35, 0x91, - 0xd4, 0x8b, 0xed, 0x10, 0x82, 0x51, 0xcb, 0xf7, 0x9f, 0xc8, 0x93, 0x2c, 0x38, 0xf6, 0xb7, 0xfa, - 0x13, 0x09, 0x4c, 0x20, 0x1c, 0xb6, 0x7c, 0x2f, 0xc4, 0x50, 0x06, 0x67, 0x6b, 0xed, 0x7a, 0x1d, - 0x87, 0x21, 0xcb, 0xf1, 0x04, 0x8a, 0x1f, 0xe1, 0x79, 0x30, 0x5e, 0x23, 0x0e, 0x69, 0x87, 0xac, - 0xbe, 0x93, 0x28, 0x7a, 0x12, 0xea, 0x9e, 0x3b, 0xad, 0xee, 0x6f, 0x24, 0xeb, 0xc9, 0x72, 0x39, - 0x75, 0x77, 0x21, 0x02, 0x8b, 0x2e, 0x94, 0x00, 0xaa, 0x9f, 0x4e, 0xc7, 0x0b, 0xc0, 0x3b, 0x60, - 0xc2, 0x20, 0xf5, 0x86, 0x71, 0x8c, 0xeb, 0x7c, 0x07, 0x14, 0xce, 0x75, 0x3b, 0x4a, 0xfe, 0xc4, - 0x39, 0x6a, 0xde, 0x57, 0x31, 0xa9, 0x37, 0x34, 0x7c, 0x8c, 0xeb, 0x2a, 0xea, 0xa1, 0xe0, 0x3d, - 0x30, 0xa9, 0x1f, 0x60, 0x8f, 0xe8, 0x8d, 0x46, 0x20, 0x4f, 0x31, 0xca, 0x62, 0xb7, 0xa3, 0xcc, - 0x73, 0x8a, 0x43, 0x5d, 0x9a, 0xd3, 0x68, 0x04, 0x2a, 0xea, 0xe3, 0xa0, 0x09, 0xe6, 0xd7, 0x1d, - 0xb7, 0xd9, 0xf2, 0x5d, 0x8f, 0x6c, 0x5a, 0x56, 0x95, 0x91, 0xa7, 0x19, 0x79, 0xa9, 0xdb, 0x51, - 0x2e, 0x71, 0xf2, 0x7e, 0x0c, 0xd1, 0x0e, 0x09, 0x69, 0x45, 0x2a, 0x59, 0x22, 0xd4, 0xc0, 0xd9, - 0x82, 0x13, 0xe2, 0x35, 0x37, 0x90, 0x31, 0xd3, 0x58, 0xe8, 0x76, 0x94, 0x39, 0xae, 0xb1, 0xe7, - 0x84, 0x58, 0x6b, 0xb8, 0x81, 0x8a, 0x62, 0x0c, 0xdc, 0x00, 0x73, 0x34, 0x7a, 0xbe, 0x5b, 0xab, - 0x81, 0x7f, 0x7c, 0x22, 0x7f, 0xc2, 0x2a, 0x51, 0xb8, 0xd2, 0xed, 0x28, 0xb2, 0xf0, 0xae, 0x75, - 0x06, 0xd1, 0x5a, 0x14, 0xa3, 0xa2, 0x34, 0x0b, 0xea, 0x60, 0x86, 0x9a, 0xaa, 0x18, 0x07, 0x5c, - 0xe6, 0x53, 0x2e, 0x73, 0xa9, 0xdb, 0x51, 0xce, 0x0b, 0x32, 0x2d, 0x8c, 0x83, 0x58, 0x24, 0xc9, - 0x80, 0x55, 0x00, 0xfb, 0xaa, 0x86, 0xd7, 0x60, 0x2f, 0x26, 0x7f, 0xcc, 0xea, 0x5f, 0x50, 0xba, - 0x1d, 0xe5, 0x72, 0x36, 0x1c, 0x1c, 0xc1, 0x54, 0x34, 0x80, 0x0b, 0xff, 0x1f, 0x8c, 0x52, 0xab, - 0xfc, 0x33, 0x7e, 0x46, 0x4c, 0x45, 0xe5, 0xa7, 0xb6, 0xc2, 0x5c, 0xb7, 0xa3, 0x4c, 0xf5, 0x05, - 0x55, 0xc4, 0xa0, 0xb0, 0x00, 0x16, 0xe9, 0xbf, 0x15, 0xaf, 0xbf, 0x99, 0x43, 0xe2, 0x07, 0x58, - 0xfe, 0x79, 0x56, 0x03, 0x0d, 0x86, 0xc2, 0x35, 0x30, 0xcb, 0x03, 0x29, 0xe2, 0x80, 0xac, 0x39, - 0xc4, 0x91, 0xbf, 0xc3, 0xbe, 0xf9, 0xc2, 0xe5, 0x6e, 0x47, 0xb9, 0xc0, 0xd7, 0x8c, 0xe2, 0xaf, - 0xe3, 0x80, 0x68, 0x0d, 0x87, 0x38, 0x2a, 0x4a, 0x71, 0x92, 0x2a, 0xec, 0xe0, 0xf8, 0xee, 0xa9, - 0x2a, 0x2d, 0x87, 0x1c, 0x26, 0x54, 0xd8, 0xc1, 0xa2, 0x83, 0x19, 0x6e, 0xd9, 0xc2, 0x27, 0x2c, - 0x94, 0xef, 0x71, 0x11, 0xa1, 0x2e, 0x91, 0xc8, 0x13, 0x7c, 0x12, 0x45, 0x92, 0x64, 0x24, 0x24, - 0x58, 0x1c, 0xdf, 0x3f, 0x4d, 0x82, 0x87, 0x91, 0x64, 0x40, 0x0b, 0x2c, 0x70, 0x83, 0x15, 0xb4, - 0x43, 0x82, 0x1b, 0x45, 0x9d, 0xc5, 0xf2, 0x03, 0x2e, 0x74, 0xad, 0xdb, 0x51, 0xae, 0x26, 0x84, - 0x08, 0x87, 0x69, 0x75, 0x27, 0x0a, 0x69, 0x10, 0x7d, 0x80, 0x2a, 0x0b, 0xef, 0x87, 0x2f, 0xa0, - 0xca, 0xa3, 0x1c, 0x44, 0x87, 0xef, 0x82, 0x69, 0xba, 0x27, 0x7b, 0xb5, 0xfb, 0x3b, 0x97, 0xbb, - 0xd8, 0xed, 0x28, 0x8b, 0x5c, 0x8e, 0xed, 0x61, 0xa1, 0x72, 0x09, 0xbc, 0xc8, 0x67, 0xe1, 0xfc, - 0xe3, 0x14, 0x3e, 0x0f, 0x23, 0x81, 0x87, 0x6f, 0x83, 0x29, 0xfa, 0x1c, 0xd7, 0xeb, 0x9f, 0x9c, - 0x2e, 0x77, 0x3b, 0xca, 0x39, 0x81, 0xde, 0xaf, 0x96, 0x88, 0x16, 0xc8, 0x6c, 0xed, 0x7f, 0x0d, - 0x27, 0xf3, 0xa5, 0x45, 0x34, 0x2c, 0x83, 0x79, 0xfa, 0x98, 0xac, 0xd1, 0xbf, 0x73, 0xe9, 0xef, - 0x8f, 0x49, 0x64, 0x2a, 0x94, 0xa5, 0x66, 0xf4, 0x58, 0x48, 0xff, 0x79, 0xae, 0x1e, 0x8f, 0x2c, - 0x4b, 0x85, 0xef, 0xa4, 0x1a, 0xe9, 0xef, 0x47, 0xd3, 0x6f, 0x17, 0x46, 0xee, 0x38, 0xb1, 0x89, - 0x1e, 0xfb, 0x66, 0xaa, 0x27, 0xfc, 0xe1, 0x45, 0x9b, 0x02, 0x7c, 0x1d, 0x80, 0xde, 0x49, 0x1b, - 0xca, 0xbf, 0x1c, 0x4b, 0x9f, 0xec, 0xbd, 0xc3, 0x39, 0x54, 0x91, 0x80, 0x54, 0x7f, 0x31, 0x1d, - 0x8f, 0x1f, 0xf4, 0x5c, 0xa6, 0x39, 0xa1, 0xe7, 0xb2, 0x94, 0x3e, 0x97, 0x69, 0x02, 0xa3, 0x73, - 0x39, 0xc2, 0xc0, 0x57, 0xc1, 0xd9, 0x32, 0x26, 0x1f, 0xfa, 0xc1, 0x13, 0xde, 0xff, 0x0a, 0xb0, - 0xdb, 0x51, 0x66, 0x39, 0xdc, 0xe3, 0x0e, 0x15, 0xc5, 0x10, 0x78, 0x1d, 0x8c, 0xb2, 0xae, 0xc1, - 0x53, 0x2b, 0x9c, 0x6c, 0xbc, 0x4d, 0x30, 0x27, 0x2c, 0x82, 0xd9, 0x35, 0xdc, 0x74, 0x4e, 0x4c, - 0x87, 0x60, 0xaf, 0x7e, 0xb2, 0x1d, 0xb2, 0x0e, 0x35, 0x23, 0x1e, 0x27, 0x0d, 0xea, 0xd7, 0x9a, - 0x1c, 0xa0, 0x1d, 0x85, 0x2a, 0x4a, 0x51, 0xe0, 0xd7, 0x41, 0x3e, 0x69, 0x41, 0x4f, 0x59, 0xaf, - 0x9a, 0x11, 0x7b, 0x55, 0x5a, 0x46, 0x0b, 0x9e, 0xaa, 0x28, 0xc3, 0x83, 0xef, 0x83, 0xc5, 0x9d, - 0x56, 0xc3, 0x21, 0xb8, 0x91, 0x8a, 0x6b, 0x86, 0x09, 0x5e, 0xef, 0x76, 0x14, 0x85, 0x0b, 0xb6, - 0x39, 0x4c, 0xcb, 0xc6, 0x37, 0x58, 0x81, 0x16, 0x0c, 0xf9, 0x6d, 0xaf, 0x61, 0xba, 0x47, 0x2e, - 0x91, 0x17, 0x97, 0xa5, 0x95, 0xb1, 0xc2, 0xf9, 0x6e, 0x47, 0x81, 0x5c, 0x2f, 0xa0, 0x3e, 0xad, - 0x49, 0x9d, 0x2a, 0x12, 0x90, 0xb0, 0x00, 0x66, 0x8d, 0x63, 0x97, 0x54, 0xbc, 0xa2, 0x13, 0x62, - 0x5a, 0x48, 0xf9, 0x7c, 0xa6, 0x8b, 0x1d, 0xbb, 0x44, 0xf3, 0x3d, 0x8d, 0xd6, 0xbc, 0x1d, 0x60, - 0x15, 0xa5, 0x18, 0xf0, 0x2d, 0x30, 0x65, 0x78, 0xce, 0x5e, 0x13, 0x57, 0x5b, 0x81, 0xbf, 0x2f, - 0x5f, 0x60, 0x02, 0x17, 0xba, 0x1d, 0x65, 0x21, 0x12, 0x60, 0x4e, 0xad, 0x45, 0xbd, 0x2a, 0x12, - 0xb1, 0xf0, 0x3e, 0x98, 0xa2, 0x32, 0xec, 0x65, 0xb6, 0x43, 0x59, 0x61, 0x79, 0x10, 0xb6, 0x77, - 0x9d, 0x35, 0x70, 0x96, 0x04, 0xfa, 0xf2, 0x22, 0x98, 0x2e, 0x4b, 0x1f, 0x6b, 0x87, 0xed, 0xfd, - 0xfd, 0x26, 0x96, 0x97, 0xd3, 0xcb, 0x32, 0x6e, 0xc8, 0xbd, 0x11, 0x35, 0xc2, 0xc2, 0x97, 0xc1, - 0x18, 0x7d, 0x0c, 0xe5, 0x6b, 0x74, 0x82, 0x2d, 0xe4, 0xbb, 0x1d, 0x65, 0xba, 0x4f, 0x0a, 0x55, - 0xc4, 0xdd, 0x70, 0x4b, 0x98, 0x54, 0x8a, 0xfe, 0xd1, 0x91, 0xe3, 0x35, 0x42, 0x59, 0x65, 0x9c, - 0xab, 0xdd, 0x8e, 0x72, 0x31, 0x3d, 0xa9, 0xd4, 0x23, 0x8c, 0x38, 0xa8, 0xc4, 0x3c, 0xba, 0x1d, - 0x51, 0xdb, 0xf3, 0x70, 0x40, 0x27, 0x27, 0xf6, 0x39, 0xdf, 0x4c, 0x77, 0xb7, 0x80, 0xf9, 0xd9, - 0x94, 0x15, 0x77, 0xb7, 0x24, 0x05, 0x96, 0x40, 0xde, 0x38, 0x26, 0x38, 0xf0, 0x9c, 0x66, 0x4f, - 0x66, 0x95, 0xc9, 0x08, 0x01, 0xe1, 0x08, 0x21, 0x0a, 0x65, 0x68, 0xb0, 0x08, 0x26, 0x6b, 0x24, - 0xc0, 0x61, 0x88, 0x83, 0x50, 0xc6, 0xcb, 0xb9, 0x95, 0xa9, 0xbb, 0x73, 0xf1, 0xc9, 0x10, 0xd9, - 0xc5, 0xf9, 0x2f, 0x8c, 0xb1, 0x2a, 0xea, 0xf3, 0xe0, 0x6d, 0x30, 0x51, 0x3c, 0xc4, 0xf5, 0x27, - 0x54, 0x63, 0x9f, 0x25, 0x46, 0xf8, 0xcc, 0xeb, 0x91, 0x47, 0x45, 0x3d, 0x10, 0xed, 0xad, 0x9c, - 0xbd, 0x85, 0x4f, 0xd8, 0x1c, 0xcf, 0xa6, 0xaf, 0x31, 0x71, 0xc3, 0xf1, 0x95, 0xd8, 0x99, 0x1d, - 0xba, 0x1f, 0x61, 0x15, 0x25, 0x19, 0xf0, 0x21, 0x80, 0x09, 0x83, 0xe9, 0x04, 0x07, 0x98, 0x8f, - 0x5f, 0x63, 0x85, 0xe5, 0x6e, 0x47, 0xb9, 0x32, 0x50, 0x47, 0x6b, 0x52, 0x9c, 0x8a, 0x06, 0x90, - 0xe1, 0x23, 0x70, 0xae, 0x6f, 0x6d, 0xef, 0xef, 0xbb, 0xc7, 0xc8, 0xf1, 0x0e, 0xb0, 0xfc, 0x19, - 0x17, 0x55, 0xbb, 0x1d, 0x65, 0x29, 0x2b, 0xca, 0x80, 0x5a, 0x40, 0x91, 0x2a, 0x1a, 0x28, 0x00, - 0x1d, 0x70, 0x61, 0x90, 0xdd, 0x3a, 0xf6, 0xe4, 0xcf, 0xb9, 0xf6, 0xcb, 0xdd, 0x8e, 0xa2, 0x9e, - 0xaa, 0xad, 0x91, 0x63, 0x4f, 0x45, 0xc3, 0x74, 0xe0, 0x26, 0x98, 0xeb, 0xb9, 0xac, 0x63, 0xaf, - 0xd2, 0x0a, 0xe5, 0x2f, 0xb8, 0xb4, 0xb0, 0x25, 0x04, 0x69, 0x72, 0xec, 0x69, 0x7e, 0x2b, 0x54, - 0x51, 0x9a, 0x06, 0xdf, 0x8b, 0x6b, 0xc3, 0xa7, 0x84, 0x90, 0x8f, 0xa2, 0x63, 0x62, 0x27, 0x8f, - 0x74, 0xf8, 0x7c, 0x11, 0xf6, 0x4a, 0x13, 0x11, 0xe0, 0x6b, 0xf1, 0x9e, 0x7a, 0x58, 0xad, 0xf1, - 0x21, 0x74, 0x4c, 0x6c, 0x1b, 0x11, 0xfb, 0x83, 0x56, 0x7f, 0x13, 0x3d, 0xac, 0xd6, 0xd4, 0x6f, - 0x80, 0x89, 0x78, 0x47, 0xd1, 0x93, 0xdd, 0x3a, 0x69, 0x45, 0x37, 0x50, 0xf1, 0x64, 0x27, 0x27, - 0x2d, 0xac, 0x22, 0xe6, 0x84, 0x37, 0xc1, 0xf8, 0x23, 0xec, 0x1e, 0x1c, 0x12, 0xd6, 0x2b, 0xa4, - 0xc2, 0x7c, 0xb7, 0xa3, 0xcc, 0x70, 0xd8, 0x87, 0xcc, 0xae, 0xa2, 0x08, 0xa0, 0x7e, 0x6b, 0x8e, - 0x8f, 0xc4, 0x54, 0xb8, 0x7f, 0xb5, 0x15, 0x85, 0x3d, 0xe7, 0x88, 0x0a, 0xb3, 0x5b, 0xae, 0xd0, - 0xb4, 0x46, 0x5e, 0xa0, 0x69, 0xad, 0x82, 0xf1, 0x47, 0xba, 0x49, 0xd1, 0xb9, 0x74, 0xcf, 0xfa, - 0xd0, 0x69, 0x72, 0x70, 0x84, 0x80, 0x15, 0xb0, 0xb0, 0x89, 0x9d, 0x80, 0xec, 0x61, 0x87, 0x94, - 0x3c, 0x82, 0x83, 0xa7, 0x4e, 0x33, 0x6a, 0x49, 0x39, 0xb1, 0x52, 0x87, 0x31, 0x48, 0x73, 0x23, - 0x94, 0x8a, 0x06, 0x31, 0x61, 0x09, 0xcc, 0x1b, 0x4d, 0x5c, 0x27, 0xae, 0xef, 0x59, 0xee, 0x11, - 0xf6, 0xdb, 0x64, 0x3b, 0x64, 0xad, 0x29, 0x27, 0x1e, 0x29, 0x38, 0x82, 0x68, 0x84, 0x63, 0x54, - 0x94, 0x65, 0xd1, 0x53, 0xc5, 0x74, 0x43, 0x82, 0x3d, 0xe1, 0x72, 0xbf, 0x98, 0x3e, 0xe6, 0x9a, - 0x0c, 0x11, 0xdf, 0x43, 0xda, 0x41, 0x33, 0x54, 0x51, 0x86, 0x06, 0x11, 0x58, 0xd0, 0x1b, 0x4f, - 0x71, 0x40, 0xdc, 0x10, 0x0b, 0x6a, 0xe7, 0x99, 0x9a, 0xf0, 0x71, 0x3a, 0x31, 0x28, 0x29, 0x38, - 0x88, 0x0c, 0xdf, 0x8a, 0xe7, 0x71, 0xbd, 0x4d, 0x7c, 0xcb, 0xac, 0x45, 0x2d, 0x46, 0xa8, 0x8d, - 0xd3, 0x26, 0xbe, 0x46, 0xa8, 0x40, 0x12, 0x49, 0x0f, 0xdd, 0xfe, 0xfd, 0x40, 0x6f, 0x93, 0x43, - 0x59, 0x66, 0xdc, 0x21, 0x57, 0x0a, 0xa7, 0x9d, 0xba, 0x52, 0x50, 0x0a, 0xfc, 0x9a, 0x28, 0xb2, - 0xee, 0x36, 0xb1, 0x7c, 0x31, 0x7d, 0x3b, 0x66, 0xec, 0x7d, 0x97, 0x76, 0x9a, 0x14, 0xb6, 0x1f, - 0xfd, 0x16, 0x3e, 0x61, 0xe4, 0x4b, 0xe9, 0x9d, 0x45, 0xbf, 0x4a, 0xce, 0x4d, 0x22, 0xa1, 0x99, - 0x99, 0xf7, 0x99, 0xc0, 0xe5, 0xf4, 0x6d, 0x44, 0x98, 0x25, 0xb9, 0xce, 0x20, 0x1a, 0xcd, 0x05, - 0x2f, 0x17, 0x1d, 0x34, 0x59, 0x55, 0x14, 0x56, 0x15, 0x21, 0x17, 0x51, 0x8d, 0xd9, 0x80, 0xca, - 0x0b, 0x92, 0xa2, 0x40, 0x0b, 0xcc, 0xf7, 0x4a, 0xd4, 0xd3, 0x59, 0x66, 0x3a, 0xc2, 0x49, 0xe6, - 0x7a, 0x2e, 0x71, 0x9d, 0xa6, 0xd6, 0xaf, 0xb2, 0x20, 0x99, 0x15, 0xa0, 0x73, 0x00, 0xfd, 0x3b, - 0xae, 0xef, 0x35, 0x56, 0xa3, 0xf4, 0x10, 0xdf, 0x2f, 0xb2, 0x08, 0xa6, 0xb7, 0x68, 0x76, 0x9d, - 0x48, 0x96, 0x59, 0x65, 0x12, 0xc2, 0x86, 0xe3, 0x77, 0x90, 0x4c, 0xad, 0x07, 0x70, 0xe9, 0xd8, - 0x1d, 0x5f, 0x50, 0x58, 0xbe, 0xaf, 0x0f, 0xbf, 0xcf, 0xf0, 0x74, 0x27, 0xe0, 0xf1, 0xcb, 0xc4, - 0xe5, 0x7e, 0x69, 0xe8, 0x8d, 0x84, 0x93, 0x45, 0x30, 0xdc, 0x4e, 0xdd, 0x20, 0x98, 0xc2, 0x8d, - 0xe7, 0x5d, 0x20, 0xb8, 0x50, 0x96, 0x49, 0xc7, 0xbb, 0x12, 0x2f, 0x45, 0xb1, 0xd9, 0x66, 0xff, - 0x2b, 0x78, 0x33, 0xbd, 0x77, 0xe2, 0x52, 0xd5, 0x39, 0x40, 0x45, 0x29, 0x06, 0xfd, 0xa2, 0x93, - 0x96, 0x1a, 0x71, 0x08, 0x8e, 0xa6, 0x0e, 0x21, 0xc1, 0x29, 0x21, 0x2d, 0xa4, 0x30, 0x15, 0x0d, - 0x22, 0x67, 0x35, 0x2d, 0xff, 0x09, 0xf6, 0xe4, 0x57, 0x9e, 0xa7, 0x49, 0x28, 0x2c, 0xa3, 0xc9, - 0xc8, 0xf0, 0x01, 0x98, 0x89, 0xef, 0x30, 0x45, 0xbf, 0xed, 0x11, 0xf9, 0x1e, 0x3b, 0x0b, 0xc5, - 0xe6, 0x15, 0x5f, 0x96, 0xea, 0xd4, 0x4f, 0x9b, 0x97, 0x88, 0x87, 0x26, 0x98, 0x7f, 0xd8, 0xf6, - 0x89, 0x53, 0x70, 0xea, 0x4f, 0xb0, 0xd7, 0x28, 0x9c, 0x10, 0x1c, 0xca, 0xaf, 0x31, 0x11, 0x61, - 0xd6, 0xff, 0x80, 0x42, 0xb4, 0x3d, 0x8e, 0xd1, 0xf6, 0x28, 0x48, 0x45, 0x59, 0x22, 0x6d, 0x25, - 0xd5, 0x00, 0xef, 0xfa, 0x04, 0xcb, 0x0f, 0xd2, 0xc7, 0x55, 0x2b, 0xc0, 0xda, 0x53, 0x9f, 0x66, - 0x27, 0xc6, 0x88, 0x19, 0xf1, 0x83, 0xa0, 0xdd, 0x22, 0x6c, 0x62, 0x92, 0xdf, 0x4b, 0x6f, 0xe3, - 0x5e, 0x46, 0x38, 0x4a, 0x63, 0x33, 0x96, 0x90, 0x11, 0x81, 0x4c, 0xdb, 0xa4, 0xe9, 0x1f, 0x1c, - 0xe0, 0x40, 0xde, 0x60, 0x89, 0x15, 0xda, 0x64, 0x93, 0xd9, 0x55, 0x14, 0x01, 0xe8, 0xfd, 0xc1, - 0xf4, 0x0f, 0x2a, 0x6d, 0xd2, 0x6a, 0x93, 0x50, 0xde, 0x64, 0xdf, 0xb3, 0x70, 0x7f, 0x68, 0xfa, - 0x07, 0x9a, 0xcf, 0x9d, 0x2a, 0x12, 0x90, 0xf0, 0x0e, 0x98, 0x30, 0xfd, 0x03, 0x13, 0x3f, 0xc5, - 0x4d, 0xb9, 0x94, 0x3e, 0x14, 0x29, 0xab, 0x49, 0x5d, 0x2a, 0xea, 0xa1, 0x56, 0xff, 0x2b, 0x81, - 0xe9, 0xb8, 0xdb, 0xb3, 0x66, 0x0e, 0xc1, 0xec, 0xd6, 0xae, 0xfd, 0x08, 0x95, 0x2c, 0xc3, 0xae, - 0x6d, 0xeb, 0xa6, 0x99, 0x3f, 0x93, 0xb0, 0x99, 0x3a, 0xda, 0x30, 0xf2, 0x12, 0x5c, 0x00, 0x73, - 0x5b, 0xbb, 0x36, 0x32, 0xf4, 0x35, 0xbb, 0x52, 0x36, 0xec, 0x2d, 0xe3, 0xfd, 0xfc, 0x08, 0x9c, - 0x07, 0x33, 0xb1, 0x11, 0xe9, 0xe5, 0x0d, 0x23, 0x9f, 0x83, 0x8b, 0x60, 0x7e, 0x6b, 0xd7, 0x5e, - 0x33, 0x4c, 0xc3, 0x32, 0x7a, 0xc8, 0xd1, 0x88, 0x1e, 0x99, 0x39, 0x76, 0x0c, 0x5e, 0x00, 0x0b, - 0x5b, 0xbb, 0xb6, 0xf5, 0xb8, 0x1c, 0xad, 0xc5, 0xdd, 0xf9, 0x71, 0x38, 0x09, 0xc6, 0x4c, 0x43, - 0xaf, 0x19, 0x79, 0x40, 0x89, 0x86, 0x69, 0x14, 0xad, 0x52, 0xa5, 0x6c, 0xa3, 0x9d, 0x72, 0xd9, - 0x40, 0xf9, 0x73, 0x30, 0x0f, 0xa6, 0x1f, 0xe9, 0x56, 0x71, 0x33, 0xb6, 0x28, 0x74, 0x59, 0xb3, - 0x52, 0xdc, 0xb2, 0x91, 0x5e, 0x34, 0x50, 0x6c, 0xbe, 0x49, 0x81, 0x4c, 0x28, 0xb6, 0xdc, 0x5b, - 0x2d, 0x80, 0xb3, 0xd1, 0x34, 0x0c, 0xa7, 0xc0, 0xd9, 0xad, 0x5d, 0x7b, 0x53, 0xaf, 0x6d, 0xe6, - 0xcf, 0xf4, 0x91, 0xc6, 0xe3, 0x6a, 0x09, 0xd1, 0x37, 0x06, 0x60, 0x3c, 0x62, 0x8d, 0xc0, 0x69, - 0x30, 0x51, 0xae, 0xd8, 0xc5, 0x4d, 0xa3, 0xb8, 0x95, 0xcf, 0xad, 0xfe, 0x38, 0x27, 0xfc, 0x7a, - 0x00, 0xe7, 0xc0, 0x54, 0xb9, 0x62, 0xd9, 0x35, 0x4b, 0x47, 0x96, 0xb1, 0x96, 0x3f, 0x03, 0xcf, - 0x03, 0x58, 0x2a, 0x97, 0xac, 0x92, 0x6e, 0x72, 0xa3, 0x6d, 0x58, 0xc5, 0xb5, 0x3c, 0xa0, 0x4b, - 0x20, 0x43, 0xb0, 0x4c, 0x51, 0x4b, 0xad, 0xb4, 0x61, 0x19, 0x68, 0x9b, 0x5b, 0xce, 0xc1, 0x65, - 0x70, 0xa5, 0x56, 0xda, 0x78, 0xb8, 0x53, 0xe2, 0x18, 0x5b, 0x2f, 0xaf, 0xd9, 0xc8, 0xd8, 0xae, - 0xec, 0x1a, 0xf6, 0x9a, 0x6e, 0xe9, 0xf9, 0x45, 0x9a, 0xf3, 0x9a, 0xbe, 0x6b, 0xd8, 0xb5, 0xb2, - 0x5e, 0xad, 0x6d, 0x56, 0xac, 0xfc, 0x12, 0xbc, 0x06, 0xae, 0x52, 0xe1, 0x0a, 0x32, 0xec, 0x78, - 0x81, 0x75, 0x54, 0xd9, 0xee, 0x43, 0x14, 0x78, 0x11, 0x2c, 0x0e, 0x76, 0x2d, 0x53, 0x76, 0x66, - 0x49, 0x1d, 0x15, 0x37, 0x4b, 0xf1, 0x9a, 0x2b, 0xf0, 0x36, 0x78, 0xe5, 0xb4, 0xa8, 0xd8, 0x73, - 0xcd, 0xaa, 0x54, 0x6d, 0x7d, 0xc3, 0x28, 0x5b, 0xf9, 0x9b, 0xf0, 0x2a, 0xb8, 0x58, 0x30, 0xf5, - 0xe2, 0xd6, 0x66, 0xc5, 0x34, 0xec, 0xaa, 0x61, 0x20, 0xbb, 0x5a, 0x41, 0x96, 0x6d, 0x3d, 0xb6, - 0xd1, 0xe3, 0x7c, 0x03, 0x2a, 0xe0, 0xf2, 0x4e, 0x79, 0x38, 0x00, 0xc3, 0x4b, 0x60, 0x71, 0xcd, - 0x30, 0xf5, 0xf7, 0x33, 0xae, 0x67, 0x12, 0xbc, 0x02, 0x2e, 0xec, 0x94, 0x07, 0x7b, 0x3f, 0x91, - 0x56, 0xff, 0x0a, 0xc0, 0x28, 0xbd, 0x3e, 0x42, 0x19, 0x9c, 0x8b, 0x73, 0x4b, 0xb7, 0xe1, 0x7a, - 0xc5, 0x34, 0x2b, 0x8f, 0x0c, 0x94, 0x3f, 0x13, 0xbd, 0x4d, 0xc6, 0x63, 0xef, 0x94, 0xad, 0x92, - 0x69, 0x5b, 0xa8, 0xb4, 0xb1, 0x61, 0xa0, 0x7e, 0x86, 0x24, 0xfa, 0x3d, 0xc4, 0x04, 0xd3, 0xd0, - 0xd7, 0xd8, 0x8e, 0xb8, 0x09, 0x6e, 0x24, 0x6d, 0xc3, 0xe8, 0x39, 0x91, 0xfe, 0x70, 0xa7, 0x82, - 0x76, 0xb6, 0xf3, 0xa3, 0x74, 0xd3, 0xc4, 0x36, 0xfa, 0xcd, 0x8d, 0xc1, 0xeb, 0x40, 0x89, 0x53, - 0x2c, 0x64, 0x37, 0x11, 0x39, 0x80, 0xf7, 0xc1, 0xeb, 0xcf, 0x01, 0x0d, 0x8b, 0x62, 0x8a, 0x96, - 0x64, 0x00, 0x37, 0x7a, 0x9f, 0x69, 0xf8, 0x1a, 0xb8, 0x33, 0xd4, 0x3d, 0x4c, 0x74, 0x06, 0xae, - 0x83, 0xc2, 0x00, 0x16, 0x7f, 0xcb, 0xc8, 0xc2, 0xf7, 0x65, 0x24, 0x14, 0x53, 0xa3, 0x4d, 0x58, - 0x44, 0xf4, 0x2b, 0xce, 0xcf, 0xc2, 0x55, 0xf0, 0xf2, 0xd0, 0xed, 0x90, 0x4c, 0x42, 0x03, 0xea, - 0xe0, 0x9d, 0x17, 0xc3, 0x0e, 0x0b, 0x1b, 0xc3, 0x97, 0xc0, 0xf2, 0x70, 0x89, 0x28, 0x25, 0xfb, - 0xf0, 0x6d, 0xf0, 0xc6, 0xf3, 0x50, 0xc3, 0x96, 0x38, 0x38, 0x7d, 0x89, 0x68, 0x1b, 0x1c, 0xd2, - 0x6f, 0x6f, 0x38, 0x8a, 0x6e, 0x0c, 0x17, 0xfe, 0x1f, 0x50, 0x07, 0x6e, 0xf6, 0x64, 0x5a, 0x9e, - 0x49, 0xf0, 0x16, 0xb8, 0x89, 0xf4, 0xf2, 0x5a, 0x65, 0xdb, 0x7e, 0x01, 0xfc, 0x27, 0x12, 0x7c, - 0x17, 0xbc, 0xf5, 0x7c, 0xe0, 0xb0, 0x17, 0xfc, 0x54, 0x82, 0x06, 0x78, 0xef, 0x85, 0xd7, 0x1b, - 0x26, 0xf3, 0x99, 0x04, 0xaf, 0x81, 0x2b, 0x83, 0xf9, 0x51, 0x1d, 0x3e, 0x97, 0xe0, 0x0a, 0xb8, - 0x7e, 0xea, 0x4a, 0x11, 0xf2, 0x0b, 0x09, 0xbe, 0x09, 0xee, 0x9d, 0x06, 0x19, 0x16, 0xc6, 0xaf, - 0x24, 0xf8, 0x00, 0xdc, 0x7f, 0x81, 0x35, 0x86, 0x09, 0xfc, 0xfa, 0x94, 0xf7, 0x88, 0x8a, 0xfd, - 0xe5, 0xf3, 0xdf, 0x23, 0x42, 0xfe, 0x46, 0x82, 0x4b, 0xe0, 0xe2, 0x60, 0x08, 0xdd, 0x13, 0xbf, - 0x95, 0xe0, 0x0d, 0xb0, 0x7c, 0xaa, 0x12, 0x85, 0xfd, 0x4e, 0x82, 0x32, 0x58, 0x28, 0x57, 0xec, - 0x75, 0xbd, 0x64, 0xda, 0x8f, 0x4a, 0xd6, 0xa6, 0x5d, 0xb3, 0x90, 0x51, 0xab, 0xe5, 0x7f, 0x3a, - 0x42, 0x43, 0x49, 0x78, 0xca, 0x95, 0xc8, 0x69, 0xaf, 0x57, 0x90, 0x6d, 0x96, 0x76, 0x8d, 0x32, - 0x45, 0x7e, 0x3c, 0x02, 0xe7, 0x00, 0xa0, 0xb0, 0x6a, 0xa5, 0x54, 0xb6, 0x6a, 0xf9, 0x6f, 0xe7, - 0xe0, 0x0c, 0x98, 0x30, 0x1e, 0x5b, 0x06, 0x2a, 0xeb, 0x66, 0xfe, 0x6f, 0xb9, 0xbb, 0x0f, 0xc0, - 0xa4, 0x15, 0x38, 0x5e, 0xd8, 0xf2, 0x03, 0x02, 0xef, 0x8a, 0x0f, 0xb3, 0xd1, 0xff, 0x67, 0x45, - 0xbf, 0xb9, 0x5f, 0x9a, 0xeb, 0x3d, 0xf3, 0x9f, 0x63, 0xd5, 0x33, 0x2b, 0xd2, 0x1d, 0xa9, 0x70, - 0xee, 0xd9, 0x9f, 0x96, 0xce, 0x3c, 0xfb, 0x6a, 0x49, 0xfa, 0xf2, 0xab, 0x25, 0xe9, 0x8f, 0x5f, - 0x2d, 0x49, 0x3f, 0xfa, 0xf3, 0xd2, 0x99, 0xbd, 0x71, 0xf6, 0x9b, 0xfd, 0xbd, 0xff, 0x05, 0x00, - 0x00, 0xff, 0xff, 0x44, 0x6a, 0x6d, 0x90, 0xfc, 0x1f, 0x00, 0x00, + 0xf5, 0x36, 0x44, 0x49, 0x96, 0x46, 0x2f, 0x68, 0x64, 0xd9, 0xf0, 0x4b, 0x90, 0xe1, 0x38, 0x3f, + 0x59, 0x09, 0x6c, 0xff, 0xec, 0x9c, 0x3c, 0x9c, 0x26, 0x0e, 0x48, 0x41, 0x12, 0x4b, 0x88, 0xa4, + 0x87, 0x90, 0x6c, 0x77, 0x51, 0x1c, 0x88, 0x1c, 0x49, 0x3c, 0xa6, 0x00, 0x06, 0x18, 0x3a, 0x52, + 0xfe, 0x81, 0xee, 0x7a, 0x9a, 0x36, 0xed, 0x69, 0xcf, 0xe9, 0x9f, 0xd0, 0xb4, 0x8b, 0xae, 0xbb, + 0x77, 0x5e, 0x6d, 0xda, 0xae, 0xda, 0x05, 0x4f, 0x9b, 0x6e, 0xba, 0xe6, 0xe9, 0x7b, 0xd1, 0xd3, + 0x33, 0x33, 0x00, 0x39, 0x00, 0x49, 0x59, 0x2b, 0x0b, 0xf7, 0x7e, 0xdf, 0x37, 0x17, 0xf7, 0x0e, + 0xe6, 0xde, 0x31, 0xc1, 0x5c, 0xd0, 0xac, 0x36, 0x77, 0x6f, 0x07, 0xcd, 0xea, 0xad, 0x66, 0xe0, + 0x13, 0x1f, 0x8e, 0x31, 0xc3, 0x25, 0x7d, 0xbf, 0x4e, 0x0e, 0x5a, 0xbb, 0xb7, 0xaa, 0xfe, 0xe1, + 0xed, 0x7d, 0x7f, 0xdf, 0xbf, 0xcd, 0xbc, 0xbb, 0xad, 0x3d, 0xf6, 0xc4, 0x1e, 0xd8, 0x5f, 0x9c, + 0xa5, 0x7d, 0x47, 0x02, 0x67, 0x11, 0x7e, 0xbf, 0x85, 0x43, 0x02, 0x6f, 0x81, 0xc9, 0x52, 0x13, + 0x07, 0x2e, 0xa9, 0xfb, 0x9e, 0x22, 0x2d, 0x4b, 0x2b, 0xb3, 0x77, 0xe5, 0x5b, 0x4c, 0xf5, 0x56, + 0xd7, 0x8e, 0x7a, 0x10, 0x78, 0x03, 0x8c, 0x6f, 0xe1, 0xc3, 0x5d, 0x1c, 0x28, 0x23, 0xcb, 0xd2, + 0xca, 0xd4, 0xdd, 0x99, 0x08, 0xcc, 0x8d, 0x28, 0x72, 0x52, 0x98, 0x8d, 0x43, 0x82, 0x03, 0x25, + 0x93, 0x80, 0x71, 0x23, 0x8a, 0x9c, 0xda, 0x5f, 0x47, 0xc0, 0x74, 0xc5, 0x73, 0x9b, 0xe1, 0x81, + 0x4f, 0xf2, 0xde, 0x9e, 0x0f, 0x97, 0x00, 0xe0, 0x0a, 0x45, 0xf7, 0x10, 0xb3, 0x78, 0x26, 0x91, + 0x60, 0x81, 0xab, 0x40, 0xe6, 0x4f, 0xb9, 0x46, 0x1d, 0x7b, 0x64, 0x1b, 0x59, 0xa1, 0x32, 0xb2, + 0x9c, 0x59, 0x99, 0x44, 0x7d, 0x76, 0xa8, 0xf5, 0xb4, 0xcb, 0x2e, 0x39, 0x60, 0x91, 0x4c, 0xa2, + 0x84, 0x8d, 0xea, 0xc5, 0xcf, 0xeb, 0xf5, 0x06, 0xae, 0xd4, 0x3f, 0xc4, 0xca, 0x28, 0xc3, 0xf5, + 0xd9, 0xe1, 0xab, 0x60, 0x3e, 0xb6, 0xd9, 0x3e, 0x71, 0x1b, 0x0c, 0x3c, 0xc6, 0xc0, 0xfd, 0x0e, + 0x51, 0x99, 0x19, 0x0b, 0xf8, 0x58, 0x19, 0x5f, 0x96, 0x56, 0x32, 0xa8, 0xcf, 0x2e, 0x46, 0xba, + 0xe9, 0x86, 0x07, 0xca, 0x59, 0x86, 0x4b, 0xd8, 0x44, 0x3d, 0x84, 0x9f, 0xd5, 0x43, 0x5a, 0xaf, + 0x89, 0xa4, 0x5e, 0x6c, 0x87, 0x10, 0x8c, 0xda, 0xbe, 0xff, 0x54, 0x99, 0x64, 0xc1, 0xb1, 0xbf, + 0xb5, 0x9f, 0x4a, 0x60, 0x02, 0xe1, 0xb0, 0xe9, 0x7b, 0x21, 0x86, 0x0a, 0x38, 0x5b, 0x69, 0x55, + 0xab, 0x38, 0x0c, 0x59, 0x8e, 0x27, 0x50, 0xfc, 0x08, 0xcf, 0x83, 0xf1, 0x0a, 0x71, 0x49, 0x2b, + 0x64, 0xf5, 0x9d, 0x44, 0xd1, 0x93, 0x50, 0xf7, 0xcc, 0x49, 0x75, 0x7f, 0x23, 0x59, 0x4f, 0x96, + 0xcb, 0xa9, 0xbb, 0x0b, 0x11, 0x58, 0x74, 0xa1, 0x04, 0x50, 0xfb, 0x6c, 0x3a, 0x5e, 0x00, 0xde, + 0x01, 0x13, 0x26, 0xa9, 0xd6, 0xcc, 0x23, 0x5c, 0xe5, 0x3b, 0x20, 0x7b, 0xae, 0xd3, 0x56, 0xe5, + 0x63, 0xf7, 0xb0, 0x71, 0x5f, 0xc3, 0xa4, 0x5a, 0xd3, 0xf1, 0x11, 0xae, 0x6a, 0xa8, 0x8b, 0x82, + 0xf7, 0xc0, 0xa4, 0xb1, 0x8f, 0x3d, 0x62, 0xd4, 0x6a, 0x81, 0x32, 0xc5, 0x28, 0x8b, 0x9d, 0xb6, + 0x3a, 0xcf, 0x29, 0x2e, 0x75, 0xe9, 0x6e, 0xad, 0x16, 0x68, 0xa8, 0x87, 0x83, 0x16, 0x98, 0x5f, + 0x77, 0xeb, 0x8d, 0xa6, 0x5f, 0xf7, 0xc8, 0xa6, 0x6d, 0x97, 0x19, 0x79, 0x9a, 0x91, 0x97, 0x3a, + 0x6d, 0xf5, 0x12, 0x27, 0xef, 0xc5, 0x10, 0xfd, 0x80, 0x90, 0x66, 0xa4, 0xd2, 0x4f, 0x84, 0x3a, + 0x38, 0x9b, 0x75, 0x43, 0xbc, 0x56, 0x0f, 0x14, 0xcc, 0x34, 0x16, 0x3a, 0x6d, 0x75, 0x8e, 0x6b, + 0xec, 0xba, 0x21, 0xd6, 0x6b, 0xf5, 0x40, 0x43, 0x31, 0x06, 0x6e, 0x80, 0x39, 0x1a, 0x3d, 0xdf, + 0xad, 0xe5, 0xc0, 0x3f, 0x3a, 0x56, 0x3e, 0x65, 0x95, 0xc8, 0x5e, 0xe9, 0xb4, 0x55, 0x45, 0x78, + 0xd7, 0x2a, 0x83, 0xe8, 0x4d, 0x8a, 0xd1, 0x50, 0x9a, 0x05, 0x0d, 0x30, 0x43, 0x4d, 0x65, 0x8c, + 0x03, 0x2e, 0xf3, 0x19, 0x97, 0xb9, 0xd4, 0x69, 0xab, 0xe7, 0x05, 0x99, 0x26, 0xc6, 0x41, 0x2c, + 0x92, 0x64, 0xc0, 0x32, 0x80, 0x3d, 0x55, 0xd3, 0xab, 0xb1, 0x17, 0x53, 0x3e, 0x61, 0xf5, 0xcf, + 0xaa, 0x9d, 0xb6, 0x7a, 0xb9, 0x3f, 0x1c, 0x1c, 0xc1, 0x34, 0x34, 0x80, 0x0b, 0xff, 0x1f, 0x8c, + 0x52, 0xab, 0xf2, 0x73, 0x7e, 0x46, 0x4c, 0x45, 0xe5, 0xa7, 0xb6, 0xec, 0x5c, 0xa7, 0xad, 0x4e, + 0xf5, 0x04, 0x35, 0xc4, 0xa0, 0x30, 0x0b, 0x16, 0xe9, 0xbf, 0x25, 0xaf, 0xb7, 0x99, 0x43, 0xe2, + 0x07, 0x58, 0xf9, 0x45, 0xbf, 0x06, 0x1a, 0x0c, 0x85, 0x6b, 0x60, 0x96, 0x07, 0x92, 0xc3, 0x01, + 0x59, 0x73, 0x89, 0xab, 0x7c, 0xc4, 0xbe, 0xf9, 0xec, 0xe5, 0x4e, 0x5b, 0xbd, 0xc0, 0xd7, 0x8c, + 0xe2, 0xaf, 0xe2, 0x80, 0xe8, 0x35, 0x97, 0xb8, 0x1a, 0x4a, 0x71, 0x92, 0x2a, 0xec, 0xe0, 0xf8, + 0xfe, 0x89, 0x2a, 0x4d, 0x97, 0x1c, 0x24, 0x54, 0xd8, 0xc1, 0x62, 0x80, 0x19, 0x6e, 0x29, 0xe0, + 0x63, 0x16, 0xca, 0x0f, 0xb8, 0x88, 0x50, 0x97, 0x48, 0xe4, 0x29, 0x3e, 0x8e, 0x22, 0x49, 0x32, + 0x12, 0x12, 0x2c, 0x8e, 0x8f, 0x4f, 0x92, 0xe0, 0x61, 0x24, 0x19, 0xd0, 0x06, 0x0b, 0xdc, 0x60, + 0x07, 0xad, 0x90, 0xe0, 0x5a, 0xce, 0x60, 0xb1, 0xfc, 0x90, 0x0b, 0x5d, 0xeb, 0xb4, 0xd5, 0xab, + 0x09, 0x21, 0xc2, 0x61, 0x7a, 0xd5, 0x8d, 0x42, 0x1a, 0x44, 0x1f, 0xa0, 0xca, 0xc2, 0xfb, 0xd1, + 0x29, 0x54, 0x79, 0x94, 0x83, 0xe8, 0xf0, 0x5d, 0x30, 0x4d, 0xf7, 0x64, 0xb7, 0x76, 0x7f, 0xe7, + 0x72, 0x17, 0x3b, 0x6d, 0x75, 0x91, 0xcb, 0xb1, 0x3d, 0x2c, 0x54, 0x2e, 0x81, 0x17, 0xf9, 0x2c, + 0x9c, 0x7f, 0x9c, 0xc0, 0xe7, 0x61, 0x24, 0xf0, 0xf0, 0x6d, 0x30, 0x45, 0x9f, 0xe3, 0x7a, 0xfd, + 0x93, 0xd3, 0x95, 0x4e, 0x5b, 0x3d, 0x27, 0xd0, 0x7b, 0xd5, 0x12, 0xd1, 0x02, 0x99, 0xad, 0xfd, + 0xaf, 0xe1, 0x64, 0xbe, 0xb4, 0x88, 0x86, 0x45, 0x30, 0x4f, 0x1f, 0x93, 0x35, 0xfa, 0x77, 0x26, + 0xfd, 0xfd, 0x31, 0x89, 0xbe, 0x0a, 0xf5, 0x53, 0xfb, 0xf4, 0x58, 0x48, 0xff, 0x79, 0xa1, 0x1e, + 0x8f, 0xac, 0x9f, 0x0a, 0xdf, 0x49, 0x35, 0xd2, 0x3f, 0x8c, 0xa6, 0xdf, 0x2e, 0x8c, 0xdc, 0x71, + 0x62, 0x13, 0x3d, 0xf6, 0xcd, 0x54, 0x4f, 0xf8, 0xe3, 0x69, 0x9b, 0x02, 0x7c, 0x1d, 0x80, 0xee, + 0x49, 0x1b, 0x2a, 0xbf, 0x1a, 0x4b, 0x9f, 0xec, 0xdd, 0xc3, 0x39, 0xd4, 0x90, 0x80, 0xd4, 0x7e, + 0x39, 0x1d, 0x8f, 0x1f, 0xf4, 0x5c, 0xa6, 0x39, 0xa1, 0xe7, 0xb2, 0x94, 0x3e, 0x97, 0x69, 0x02, + 0xa3, 0x73, 0x39, 0xc2, 0xc0, 0x57, 0xc1, 0xd9, 0x22, 0x26, 0x1f, 0xf8, 0xc1, 0x53, 0xde, 0xff, + 0xb2, 0xb0, 0xd3, 0x56, 0x67, 0x39, 0xdc, 0xe3, 0x0e, 0x0d, 0xc5, 0x10, 0x78, 0x1d, 0x8c, 0xb2, + 0xae, 0xc1, 0x53, 0x2b, 0x9c, 0x6c, 0xbc, 0x4d, 0x30, 0x27, 0xcc, 0x81, 0xd9, 0x35, 0xdc, 0x70, + 0x8f, 0x2d, 0x97, 0x60, 0xaf, 0x7a, 0xbc, 0x15, 0xb2, 0x0e, 0x35, 0x23, 0x1e, 0x27, 0x35, 0xea, + 0xd7, 0x1b, 0x1c, 0xa0, 0x1f, 0x86, 0x1a, 0x4a, 0x51, 0xe0, 0x37, 0x81, 0x9c, 0xb4, 0xa0, 0x67, + 0xac, 0x57, 0xcd, 0x88, 0xbd, 0x2a, 0x2d, 0xa3, 0x07, 0xcf, 0x34, 0xd4, 0xc7, 0x83, 0x4f, 0xc0, + 0xe2, 0x76, 0xb3, 0xe6, 0x12, 0x5c, 0x4b, 0xc5, 0x35, 0xc3, 0x04, 0xaf, 0x77, 0xda, 0xaa, 0xca, + 0x05, 0x5b, 0x1c, 0xa6, 0xf7, 0xc7, 0x37, 0x58, 0x81, 0x16, 0x0c, 0xf9, 0x2d, 0xaf, 0x66, 0xd5, + 0x0f, 0xeb, 0x44, 0x59, 0x5c, 0x96, 0x56, 0xc6, 0xb2, 0xe7, 0x3b, 0x6d, 0x15, 0x72, 0xbd, 0x80, + 0xfa, 0xf4, 0x06, 0x75, 0x6a, 0x48, 0x40, 0xc2, 0x2c, 0x98, 0x35, 0x8f, 0xea, 0xa4, 0xe4, 0xe5, + 0xdc, 0x10, 0xd3, 0x42, 0x2a, 0xe7, 0xfb, 0xba, 0xd8, 0x51, 0x9d, 0xe8, 0xbe, 0xa7, 0xd3, 0x9a, + 0xb7, 0x02, 0xac, 0xa1, 0x14, 0x03, 0xbe, 0x05, 0xa6, 0x4c, 0xcf, 0xdd, 0x6d, 0xe0, 0x72, 0x33, + 0xf0, 0xf7, 0x94, 0x0b, 0x4c, 0xe0, 0x42, 0xa7, 0xad, 0x2e, 0x44, 0x02, 0xcc, 0xa9, 0x37, 0xa9, + 0x57, 0x43, 0x22, 0x16, 0xde, 0x07, 0x53, 0x54, 0x86, 0xbd, 0xcc, 0x56, 0xa8, 0xa8, 0x2c, 0x0f, + 0xc2, 0xf6, 0xae, 0xb2, 0x06, 0xce, 0x92, 0x40, 0x5f, 0x5e, 0x04, 0xd3, 0x65, 0xe9, 0x63, 0xe5, + 0xa0, 0xb5, 0xb7, 0xd7, 0xc0, 0xca, 0x72, 0x7a, 0x59, 0xc6, 0x0d, 0xb9, 0x37, 0xa2, 0x46, 0x58, + 0xf8, 0x32, 0x18, 0xa3, 0x8f, 0xa1, 0x72, 0x8d, 0x4e, 0xb0, 0x59, 0xb9, 0xd3, 0x56, 0xa7, 0x7b, + 0xa4, 0x50, 0x43, 0xdc, 0x0d, 0x0b, 0xc2, 0xa4, 0x92, 0xf3, 0x0f, 0x0f, 0x5d, 0xaf, 0x16, 0x2a, + 0x1a, 0xe3, 0x5c, 0xed, 0xb4, 0xd5, 0x8b, 0xe9, 0x49, 0xa5, 0x1a, 0x61, 0xc4, 0x41, 0x25, 0xe6, + 0xd1, 0xed, 0x88, 0x5a, 0x9e, 0x87, 0x03, 0x3a, 0x39, 0xb1, 0xcf, 0xf9, 0x66, 0xba, 0xbb, 0x05, + 0xcc, 0xcf, 0xa6, 0xac, 0xb8, 0xbb, 0x25, 0x29, 0x30, 0x0f, 0x64, 0xf3, 0x88, 0xe0, 0xc0, 0x73, + 0x1b, 0x5d, 0x99, 0x55, 0x26, 0x23, 0x04, 0x84, 0x23, 0x84, 0x28, 0xd4, 0x47, 0x83, 0x39, 0x30, + 0x59, 0x21, 0x01, 0x0e, 0x43, 0x1c, 0x84, 0x0a, 0x5e, 0xce, 0xac, 0x4c, 0xdd, 0x9d, 0x8b, 0x4f, + 0x86, 0xc8, 0x2e, 0xce, 0x7f, 0x61, 0x8c, 0xd5, 0x50, 0x8f, 0x07, 0x6f, 0x83, 0x89, 0xdc, 0x01, + 0xae, 0x3e, 0xa5, 0x1a, 0x7b, 0x2c, 0x31, 0xc2, 0x67, 0x5e, 0x8d, 0x3c, 0x1a, 0xea, 0x82, 0x68, + 0x6f, 0xe5, 0xec, 0x02, 0x3e, 0x66, 0x73, 0x3c, 0x9b, 0xbe, 0xc6, 0xc4, 0x0d, 0xc7, 0x57, 0x62, + 0x67, 0x76, 0x58, 0xff, 0x10, 0x6b, 0x28, 0xc9, 0x80, 0x0f, 0x01, 0x4c, 0x18, 0x2c, 0x37, 0xd8, + 0xc7, 0x7c, 0xfc, 0x1a, 0xcb, 0x2e, 0x77, 0xda, 0xea, 0x95, 0x81, 0x3a, 0x7a, 0x83, 0xe2, 0x34, + 0x34, 0x80, 0x0c, 0x1f, 0x81, 0x73, 0x3d, 0x6b, 0x6b, 0x6f, 0xaf, 0x7e, 0x84, 0x5c, 0x6f, 0x1f, + 0x2b, 0x9f, 0x73, 0x51, 0xad, 0xd3, 0x56, 0x97, 0xfa, 0x45, 0x19, 0x50, 0x0f, 0x28, 0x52, 0x43, + 0x03, 0x05, 0xa0, 0x0b, 0x2e, 0x0c, 0xb2, 0xdb, 0x47, 0x9e, 0xf2, 0x05, 0xd7, 0x7e, 0xb9, 0xd3, + 0x56, 0xb5, 0x13, 0xb5, 0x75, 0x72, 0xe4, 0x69, 0x68, 0x98, 0x0e, 0xdc, 0x04, 0x73, 0x5d, 0x97, + 0x7d, 0xe4, 0x95, 0x9a, 0xa1, 0xf2, 0x25, 0x97, 0x16, 0xb6, 0x84, 0x20, 0x4d, 0x8e, 0x3c, 0xdd, + 0x6f, 0x86, 0x1a, 0x4a, 0xd3, 0xe0, 0x7b, 0x71, 0x6d, 0xf8, 0x94, 0x10, 0xf2, 0x51, 0x74, 0x4c, + 0xec, 0xe4, 0x91, 0x0e, 0x9f, 0x2f, 0xc2, 0x6e, 0x69, 0x22, 0x02, 0x7c, 0x2d, 0xde, 0x53, 0x0f, + 0xcb, 0x15, 0x3e, 0x84, 0x8e, 0x89, 0x6d, 0x23, 0x62, 0xbf, 0xdf, 0xec, 0x6d, 0xa2, 0x87, 0xe5, + 0x8a, 0xf6, 0x2d, 0x30, 0x11, 0xef, 0x28, 0x7a, 0xb2, 0xdb, 0xc7, 0xcd, 0xe8, 0x06, 0x2a, 0x9e, + 0xec, 0xe4, 0xb8, 0x89, 0x35, 0xc4, 0x9c, 0xf0, 0x26, 0x18, 0x7f, 0x84, 0xeb, 0xfb, 0x07, 0x84, + 0xf5, 0x0a, 0x29, 0x3b, 0xdf, 0x69, 0xab, 0x33, 0x1c, 0xf6, 0x01, 0xb3, 0x6b, 0x28, 0x02, 0x68, + 0xdf, 0x9d, 0xe3, 0x23, 0x31, 0x15, 0xee, 0x5d, 0x6d, 0x45, 0x61, 0xcf, 0x3d, 0xa4, 0xc2, 0xec, + 0x96, 0x2b, 0x34, 0xad, 0x91, 0x53, 0x34, 0xad, 0x55, 0x30, 0xfe, 0xc8, 0xb0, 0x28, 0x3a, 0x93, + 0xee, 0x59, 0x1f, 0xb8, 0x0d, 0x0e, 0x8e, 0x10, 0xb0, 0x04, 0x16, 0x36, 0xb1, 0x1b, 0x90, 0x5d, + 0xec, 0x92, 0xbc, 0x47, 0x70, 0xf0, 0xcc, 0x6d, 0x44, 0x2d, 0x29, 0x23, 0x56, 0xea, 0x20, 0x06, + 0xe9, 0xf5, 0x08, 0xa5, 0xa1, 0x41, 0x4c, 0x98, 0x07, 0xf3, 0x66, 0x03, 0x57, 0x49, 0xdd, 0xf7, + 0xec, 0xfa, 0x21, 0xf6, 0x5b, 0x64, 0x2b, 0x64, 0xad, 0x29, 0x23, 0x1e, 0x29, 0x38, 0x82, 0xe8, + 0x84, 0x63, 0x34, 0xd4, 0xcf, 0xa2, 0xa7, 0x8a, 0x55, 0x0f, 0x09, 0xf6, 0x84, 0xcb, 0xfd, 0x62, + 0xfa, 0x98, 0x6b, 0x30, 0x44, 0x7c, 0x0f, 0x69, 0x05, 0x8d, 0x50, 0x43, 0x7d, 0x34, 0x88, 0xc0, + 0x82, 0x51, 0x7b, 0x86, 0x03, 0x52, 0x0f, 0xb1, 0xa0, 0x76, 0x9e, 0xa9, 0x09, 0x1f, 0xa7, 0x1b, + 0x83, 0x92, 0x82, 0x83, 0xc8, 0xf0, 0xad, 0x78, 0x1e, 0x37, 0x5a, 0xc4, 0xb7, 0xad, 0x4a, 0xd4, + 0x62, 0x84, 0xda, 0xb8, 0x2d, 0xe2, 0xeb, 0x84, 0x0a, 0x24, 0x91, 0xf4, 0xd0, 0xed, 0xdd, 0x0f, + 0x8c, 0x16, 0x39, 0x50, 0x14, 0xc6, 0x1d, 0x72, 0xa5, 0x70, 0x5b, 0xa9, 0x2b, 0x05, 0xa5, 0xc0, + 0x6f, 0x88, 0x22, 0xeb, 0xf5, 0x06, 0x56, 0x2e, 0xa6, 0x6f, 0xc7, 0x8c, 0xbd, 0x57, 0xa7, 0x9d, + 0x26, 0x85, 0xed, 0x45, 0x5f, 0xc0, 0xc7, 0x8c, 0x7c, 0x29, 0xbd, 0xb3, 0xe8, 0x57, 0xc9, 0xb9, + 0x49, 0x24, 0xb4, 0xfa, 0xe6, 0x7d, 0x26, 0x70, 0x39, 0x7d, 0x1b, 0x11, 0x66, 0x49, 0xae, 0x33, + 0x88, 0x46, 0x73, 0xc1, 0xcb, 0x45, 0x07, 0x4d, 0x56, 0x15, 0x95, 0x55, 0x45, 0xc8, 0x45, 0x54, + 0x63, 0x36, 0xa0, 0xf2, 0x82, 0xa4, 0x28, 0xd0, 0x06, 0xf3, 0xdd, 0x12, 0x75, 0x75, 0x96, 0x99, + 0x8e, 0x70, 0x92, 0xd5, 0xbd, 0x3a, 0xa9, 0xbb, 0x0d, 0xbd, 0x57, 0x65, 0x41, 0xb2, 0x5f, 0x80, + 0xce, 0x01, 0xf4, 0xef, 0xb8, 0xbe, 0xd7, 0x58, 0x8d, 0xd2, 0x43, 0x7c, 0xaf, 0xc8, 0x22, 0x98, + 0xde, 0xa2, 0xd9, 0x75, 0x22, 0x59, 0x66, 0x8d, 0x49, 0x08, 0x1b, 0x8e, 0xdf, 0x41, 0xfa, 0x6a, + 0x3d, 0x80, 0x4b, 0xc7, 0xee, 0xf8, 0x82, 0xc2, 0xf2, 0x7d, 0x7d, 0xf8, 0x7d, 0x86, 0xa7, 0x3b, + 0x01, 0x8f, 0x5f, 0x26, 0x2e, 0xf7, 0x4b, 0x43, 0x6f, 0x24, 0x9c, 0x2c, 0x82, 0xe1, 0x56, 0xea, + 0x06, 0xc1, 0x14, 0x6e, 0xbc, 0xe8, 0x02, 0xc1, 0x85, 0xfa, 0x99, 0x74, 0xbc, 0xcb, 0xf3, 0x52, + 0xe4, 0x1a, 0x2d, 0xf6, 0xbf, 0x82, 0x37, 0xd3, 0x7b, 0x27, 0x2e, 0x55, 0x95, 0x03, 0x34, 0x94, + 0x62, 0xd0, 0x2f, 0x3a, 0x69, 0xa9, 0x10, 0x97, 0xe0, 0x68, 0xea, 0x10, 0x12, 0x9c, 0x12, 0xd2, + 0x43, 0x0a, 0xd3, 0xd0, 0x20, 0x72, 0xbf, 0xa6, 0xed, 0x3f, 0xc5, 0x9e, 0xf2, 0xca, 0x8b, 0x34, + 0x09, 0x85, 0xf5, 0x69, 0x32, 0x32, 0x7c, 0x00, 0x66, 0xe2, 0x3b, 0x4c, 0xce, 0x6f, 0x79, 0x44, + 0xb9, 0xc7, 0xce, 0x42, 0xb1, 0x79, 0xc5, 0x97, 0xa5, 0x2a, 0xf5, 0xd3, 0xe6, 0x25, 0xe2, 0xa1, + 0x05, 0xe6, 0x1f, 0xb6, 0x7c, 0xe2, 0x66, 0xdd, 0xea, 0x53, 0xec, 0xd5, 0xb2, 0xc7, 0x04, 0x87, + 0xca, 0x6b, 0x4c, 0x44, 0x98, 0xf5, 0xdf, 0xa7, 0x10, 0x7d, 0x97, 0x63, 0xf4, 0x5d, 0x0a, 0xd2, + 0x50, 0x3f, 0x91, 0xb6, 0x92, 0x72, 0x80, 0x77, 0x7c, 0x82, 0x95, 0x07, 0xe9, 0xe3, 0xaa, 0x19, + 0x60, 0xfd, 0x99, 0x4f, 0xb3, 0x13, 0x63, 0xc4, 0x8c, 0xf8, 0x41, 0xd0, 0x6a, 0x12, 0x36, 0x31, + 0x29, 0xef, 0xa5, 0xb7, 0x71, 0x37, 0x23, 0x1c, 0xa5, 0xb3, 0x19, 0x4b, 0xc8, 0x88, 0x40, 0xa6, + 0x6d, 0xd2, 0xf2, 0xf7, 0xf7, 0x71, 0xa0, 0x6c, 0xb0, 0xc4, 0x0a, 0x6d, 0xb2, 0xc1, 0xec, 0x1a, + 0x8a, 0x00, 0xf4, 0xfe, 0x60, 0xf9, 0xfb, 0xa5, 0x16, 0x69, 0xb6, 0x48, 0xa8, 0x6c, 0xb2, 0xef, + 0x59, 0xb8, 0x3f, 0x34, 0xfc, 0x7d, 0xdd, 0xe7, 0x4e, 0x0d, 0x09, 0x48, 0x78, 0x07, 0x4c, 0x58, + 0xfe, 0xbe, 0x85, 0x9f, 0xe1, 0x86, 0x92, 0x4f, 0x1f, 0x8a, 0x94, 0xd5, 0xa0, 0x2e, 0x0d, 0x75, + 0x51, 0xab, 0xff, 0x95, 0xc0, 0x74, 0xdc, 0xed, 0x59, 0x33, 0x87, 0x60, 0xb6, 0xb0, 0xe3, 0x3c, + 0x42, 0x79, 0xdb, 0x74, 0x2a, 0x5b, 0x86, 0x65, 0xc9, 0x67, 0x12, 0x36, 0xcb, 0x40, 0x1b, 0xa6, + 0x2c, 0xc1, 0x05, 0x30, 0x57, 0xd8, 0x71, 0x90, 0x69, 0xac, 0x39, 0xa5, 0xa2, 0xe9, 0x14, 0xcc, + 0x27, 0xf2, 0x08, 0x9c, 0x07, 0x33, 0xb1, 0x11, 0x19, 0xc5, 0x0d, 0x53, 0xce, 0xc0, 0x45, 0x30, + 0x5f, 0xd8, 0x71, 0xd6, 0x4c, 0xcb, 0xb4, 0xcd, 0x2e, 0x72, 0x34, 0xa2, 0x47, 0x66, 0x8e, 0x1d, + 0x83, 0x17, 0xc0, 0x42, 0x61, 0xc7, 0xb1, 0x1f, 0x17, 0xa3, 0xb5, 0xb8, 0x5b, 0x1e, 0x87, 0x93, + 0x60, 0xcc, 0x32, 0x8d, 0x8a, 0x29, 0x03, 0x4a, 0x34, 0x2d, 0x33, 0x67, 0xe7, 0x4b, 0x45, 0x07, + 0x6d, 0x17, 0x8b, 0x26, 0x92, 0xcf, 0x41, 0x19, 0x4c, 0x3f, 0x32, 0xec, 0xdc, 0x66, 0x6c, 0x51, + 0xe9, 0xb2, 0x56, 0x29, 0x57, 0x70, 0x90, 0x91, 0x33, 0x51, 0x6c, 0xbe, 0x49, 0x81, 0x4c, 0x28, + 0xb6, 0xdc, 0x5b, 0xfd, 0x36, 0x38, 0x1b, 0x4d, 0xc3, 0x70, 0x0a, 0x9c, 0x2d, 0xec, 0x38, 0x9b, + 0x46, 0x65, 0x53, 0x3e, 0xd3, 0x43, 0x9a, 0x8f, 0xcb, 0x79, 0x44, 0xdf, 0x18, 0x80, 0xf1, 0x88, + 0x35, 0x02, 0xa7, 0xc1, 0x44, 0xb1, 0xe4, 0xe4, 0x36, 0xcd, 0x5c, 0x41, 0xce, 0xc0, 0x4b, 0xe0, + 0x7c, 0x65, 0xb3, 0x84, 0x6c, 0xc7, 0xb6, 0x2d, 0x27, 0xc1, 0x1a, 0x5d, 0xfd, 0x49, 0x46, 0xf8, + 0x65, 0x01, 0xce, 0x81, 0xa9, 0x62, 0xc9, 0x76, 0x2a, 0xb6, 0x81, 0x6c, 0x73, 0x4d, 0x3e, 0x03, + 0xcf, 0x03, 0x98, 0x2f, 0xe6, 0xed, 0xbc, 0x61, 0x71, 0xa3, 0x63, 0xda, 0xb9, 0x35, 0x19, 0xd0, + 0xe5, 0x91, 0x29, 0x58, 0xa6, 0xa8, 0xa5, 0x92, 0xdf, 0xb0, 0x4d, 0xb4, 0xc5, 0x2d, 0xe7, 0xe0, + 0x32, 0xb8, 0x52, 0xc9, 0x6f, 0x3c, 0xdc, 0xce, 0x73, 0x8c, 0x63, 0x14, 0xd7, 0x1c, 0x64, 0x6e, + 0x95, 0x76, 0x4c, 0x67, 0xcd, 0xb0, 0x0d, 0x79, 0x91, 0xd6, 0xa3, 0x62, 0xec, 0x98, 0x4e, 0xa5, + 0x68, 0x94, 0x2b, 0x9b, 0x25, 0x5b, 0x5e, 0x82, 0xd7, 0xc0, 0x55, 0x2a, 0x5c, 0x42, 0xa6, 0x13, + 0x2f, 0xb0, 0x8e, 0x4a, 0x5b, 0x3d, 0x88, 0x0a, 0x2f, 0x82, 0xc5, 0xc1, 0xae, 0x65, 0xca, 0xee, + 0x5b, 0xd2, 0x40, 0xb9, 0xcd, 0x7c, 0xbc, 0xe6, 0x0a, 0xbc, 0x0d, 0x5e, 0x39, 0x29, 0x2a, 0xf6, + 0x5c, 0xb1, 0x4b, 0x65, 0xc7, 0xd8, 0x30, 0x8b, 0xb6, 0x7c, 0x13, 0x5e, 0x05, 0x17, 0xb3, 0x96, + 0x91, 0x2b, 0x6c, 0x96, 0x2c, 0xd3, 0x29, 0x9b, 0x26, 0x72, 0xca, 0x2c, 0x97, 0x8f, 0x1d, 0xf4, + 0x58, 0xae, 0x41, 0x15, 0x5c, 0xde, 0x2e, 0x0e, 0x07, 0x60, 0x78, 0x09, 0x2c, 0xae, 0x99, 0x96, + 0xf1, 0xa4, 0xcf, 0xf5, 0x5c, 0x82, 0x57, 0xc0, 0x85, 0xed, 0xe2, 0x60, 0xef, 0xa7, 0xd2, 0xea, + 0xc7, 0x53, 0x60, 0x94, 0x5e, 0x2d, 0xa1, 0x02, 0xce, 0xc5, 0xb9, 0xa5, 0x5b, 0x74, 0xbd, 0x64, + 0x59, 0xa5, 0x47, 0x26, 0x92, 0xcf, 0x44, 0x6f, 0xd3, 0xe7, 0x71, 0xb6, 0x8b, 0x76, 0xde, 0x72, + 0x6c, 0x94, 0xdf, 0xd8, 0x30, 0x51, 0x2f, 0x43, 0x12, 0xfd, 0x56, 0x62, 0x82, 0x65, 0x1a, 0x6b, + 0x6c, 0xb7, 0xdc, 0x04, 0x37, 0x92, 0xb6, 0x61, 0xf4, 0x8c, 0x48, 0x7f, 0xb8, 0x5d, 0x42, 0xdb, + 0x5b, 0xf2, 0x28, 0xdd, 0x34, 0xb1, 0x8d, 0x7e, 0x8f, 0x63, 0xf0, 0x3a, 0x50, 0xe3, 0x14, 0x0b, + 0xd9, 0x4d, 0x44, 0x0e, 0xe0, 0x7d, 0xf0, 0xfa, 0x0b, 0x40, 0xc3, 0xa2, 0x98, 0xa2, 0x25, 0x19, + 0xc0, 0x8d, 0xde, 0x67, 0x1a, 0xbe, 0x06, 0xee, 0x0c, 0x75, 0x0f, 0x13, 0x9d, 0x81, 0xeb, 0x20, + 0x3b, 0x80, 0xc5, 0xdf, 0x32, 0xb2, 0xf0, 0x7d, 0x19, 0x09, 0xc5, 0xd4, 0x68, 0x13, 0xe6, 0x10, + 0xfd, 0xc2, 0xe5, 0x59, 0xb8, 0x0a, 0x5e, 0x1e, 0xba, 0x1d, 0x92, 0x49, 0xa8, 0x41, 0x03, 0xbc, + 0x73, 0x3a, 0xec, 0xb0, 0xb0, 0x31, 0x7c, 0x09, 0x2c, 0x0f, 0x97, 0x88, 0x52, 0xb2, 0x07, 0xdf, + 0x06, 0x6f, 0xbc, 0x08, 0x35, 0x6c, 0x89, 0xfd, 0x93, 0x97, 0x88, 0xb6, 0xc1, 0x01, 0xfd, 0xf6, + 0x86, 0xa3, 0xe8, 0xc6, 0xa8, 0xc3, 0xff, 0x03, 0xda, 0xc0, 0xcd, 0x9e, 0x4c, 0xcb, 0x73, 0x09, + 0xde, 0x02, 0x37, 0x91, 0x51, 0x5c, 0x2b, 0x6d, 0x39, 0xa7, 0xc0, 0x7f, 0x2a, 0xc1, 0x77, 0xc1, + 0x5b, 0x2f, 0x06, 0x0e, 0x7b, 0xc1, 0xcf, 0x24, 0x68, 0x82, 0xf7, 0x4e, 0xbd, 0xde, 0x30, 0x99, + 0xcf, 0x25, 0x78, 0x0d, 0x5c, 0x19, 0xcc, 0x8f, 0xea, 0xf0, 0x85, 0x04, 0x57, 0xc0, 0xf5, 0x13, + 0x57, 0x8a, 0x90, 0x5f, 0x4a, 0xf0, 0x4d, 0x70, 0xef, 0x24, 0xc8, 0xb0, 0x30, 0x7e, 0x2d, 0xc1, + 0x07, 0xe0, 0xfe, 0x29, 0xd6, 0x18, 0x26, 0xf0, 0x9b, 0x13, 0xde, 0x23, 0x2a, 0xf6, 0x57, 0x2f, + 0x7e, 0x8f, 0x08, 0xf9, 0x5b, 0x09, 0x2e, 0x81, 0x8b, 0x83, 0x21, 0x74, 0x4f, 0xfc, 0x4e, 0x82, + 0x37, 0xc0, 0xf2, 0x89, 0x4a, 0x14, 0xf6, 0x7b, 0x09, 0x2a, 0x60, 0xa1, 0x58, 0x72, 0xd6, 0x8d, + 0xbc, 0xe5, 0x3c, 0xca, 0xdb, 0x9b, 0x4e, 0xc5, 0x46, 0x66, 0xa5, 0x22, 0xff, 0x6c, 0x84, 0x86, + 0x92, 0xf0, 0x14, 0x4b, 0x91, 0xd3, 0x59, 0x2f, 0x21, 0xc7, 0xca, 0xef, 0x98, 0x45, 0x8a, 0xfc, + 0x64, 0x04, 0xce, 0x01, 0x40, 0x61, 0xe5, 0x52, 0xbe, 0x68, 0x57, 0xe4, 0xef, 0x65, 0xe0, 0x4b, + 0x40, 0xed, 0x19, 0x38, 0x7b, 0x2d, 0x5f, 0x29, 0x38, 0xf9, 0x92, 0x63, 0x19, 0xb6, 0x59, 0xcc, + 0x3d, 0x91, 0x3f, 0xca, 0xc0, 0x19, 0x30, 0x61, 0x3e, 0xb6, 0x4d, 0x54, 0x34, 0x2c, 0xf9, 0x6f, + 0x99, 0xbb, 0x0f, 0xc0, 0xa4, 0x1d, 0xb8, 0x5e, 0xd8, 0xf4, 0x03, 0x02, 0xef, 0x8a, 0x0f, 0xb3, + 0xd1, 0xff, 0x88, 0x45, 0xbf, 0xda, 0x5f, 0x9a, 0xeb, 0x3e, 0xf3, 0x1f, 0x74, 0xb5, 0x33, 0x2b, + 0xd2, 0x1d, 0x29, 0x7b, 0xee, 0xf9, 0x9f, 0x97, 0xce, 0x3c, 0xff, 0x7a, 0x49, 0xfa, 0xea, 0xeb, + 0x25, 0xe9, 0x4f, 0x5f, 0x2f, 0x49, 0x3f, 0xfe, 0xcb, 0xd2, 0x99, 0xdd, 0x71, 0xf6, 0xab, 0xff, + 0xbd, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x21, 0xcd, 0x4e, 0x90, 0x3e, 0x20, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2972,10 +2981,7 @@ func (m *Request) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthRpc - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { @@ -3275,10 +3281,7 @@ func (m *SnapshotInfo) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthRpc - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { @@ -3453,10 +3456,7 @@ func (m *Response) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthRpc - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { @@ -4263,10 +4263,7 @@ func (m *Member) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthRpc - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { @@ -4895,10 +4892,7 @@ func (m *Tester) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthRpc - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { @@ -4992,10 +4986,7 @@ func (m *Stresser) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthRpc - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { @@ -5850,10 +5841,7 @@ func (m *Etcd) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthRpc - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { diff --git a/tests/functional/rpcpb/rpc.proto b/tests/functional/rpcpb/rpc.proto index 7d0fc6f31..87ba05337 100644 --- a/tests/functional/rpcpb/rpc.proto +++ b/tests/functional/rpcpb/rpc.proto @@ -194,6 +194,7 @@ enum Checker { LEASE_EXPIRE = 1; RUNNER = 2; NO_CHECK = 3; + SHORT_TTL_LEASE_EXPIRE = 4; } message Etcd { diff --git a/tests/functional/tester/checker_lease_expire.go b/tests/functional/tester/checker_lease_expire.go index bb6e16cd4..0429b20f6 100644 --- a/tests/functional/tester/checker_lease_expire.go +++ b/tests/functional/tester/checker_lease_expire.go @@ -74,12 +74,13 @@ func (lc *leaseExpireChecker) Check() error { }() lc.cli = cli - if err := lc.check(true, lc.ls.revokedLeases.leases); err != nil { + if err := check(lc.lg, lc.cli, true, lc.ls.revokedLeases.leases); err != nil { return err } - if err := lc.check(false, lc.ls.aliveLeases.leases); err != nil { + if err := check(lc.lg, lc.cli, false, lc.ls.aliveLeases.leases); err != nil { return err } + return lc.checkShortLivedLeases() } @@ -109,7 +110,7 @@ func (lc *leaseExpireChecker) checkShortLivedLease(ctx context.Context, leaseID // retry in case of transient failure or lease is expired but not yet revoked due to the fact that etcd cluster didn't have enought time to delete it. var resp *clientv3.LeaseTimeToLiveResponse for i := 0; i < retries; i++ { - resp, err = lc.getLeaseByID(ctx, leaseID) + resp, err = getLeaseByID(ctx, lc.cli, leaseID) // lease not found, for ~v3.1 compatibilities, check ErrLeaseNotFound if (err == nil && resp.TTL == -1) || (err != nil && rpctypes.Error(err) == rpctypes.ErrLeaseNotFound) { return nil @@ -142,7 +143,7 @@ func (lc *leaseExpireChecker) checkShortLivedLease(ctx context.Context, leaseID ) time.Sleep(time.Second) } - if err = lc.checkLease(ctx, false, leaseID); err != nil { + if err = checkLease(ctx, lc.lg, lc.cli, false, leaseID); err != nil { continue } return nil @@ -150,21 +151,21 @@ func (lc *leaseExpireChecker) checkShortLivedLease(ctx context.Context, leaseID return err } -func (lc *leaseExpireChecker) checkLease(ctx context.Context, expired bool, leaseID int64) error { - keysExpired, err := lc.hasKeysAttachedToLeaseExpired(ctx, leaseID) +func checkLease(ctx context.Context, lg *zap.Logger, cli *clientv3.Client, expired bool, leaseID int64) error { + keysExpired, err := hasKeysAttachedToLeaseExpired(ctx, lg, cli, leaseID) if err != nil { - lc.lg.Warn( + lg.Warn( "hasKeysAttachedToLeaseExpired failed", - zap.String("endpoint", lc.m.EtcdClientEndpoint), + zap.Any("endpoint", cli.Endpoints()), zap.Error(err), ) return err } - leaseExpired, err := lc.hasLeaseExpired(ctx, leaseID) + leaseExpired, err := hasLeaseExpired(ctx, lg, cli, leaseID) if err != nil { - lc.lg.Warn( + lg.Warn( "hasLeaseExpired failed", - zap.String("endpoint", lc.m.EtcdClientEndpoint), + zap.Any("endpoint", cli.Endpoints()), zap.Error(err), ) return err @@ -178,11 +179,11 @@ func (lc *leaseExpireChecker) checkLease(ctx context.Context, expired bool, leas return nil } -func (lc *leaseExpireChecker) check(expired bool, leases map[int64]time.Time) error { +func check(lg *zap.Logger, cli *clientv3.Client, expired bool, leases map[int64]time.Time) error { ctx, cancel := context.WithTimeout(context.Background(), leaseExpireCheckerTimeout) defer cancel() for leaseID := range leases { - if err := lc.checkLease(ctx, expired, leaseID); err != nil { + if err := checkLease(ctx, lg, cli, expired, leaseID); err != nil { return err } } @@ -190,18 +191,18 @@ func (lc *leaseExpireChecker) check(expired bool, leases map[int64]time.Time) er } // TODO: handle failures from "grpc.WaitForReady(true)" -func (lc *leaseExpireChecker) getLeaseByID(ctx context.Context, leaseID int64) (*clientv3.LeaseTimeToLiveResponse, error) { - return lc.cli.TimeToLive( +func getLeaseByID(ctx context.Context, cli *clientv3.Client, leaseID int64) (*clientv3.LeaseTimeToLiveResponse, error) { + return cli.TimeToLive( ctx, clientv3.LeaseID(leaseID), clientv3.WithAttachedKeys(), ) } -func (lc *leaseExpireChecker) hasLeaseExpired(ctx context.Context, leaseID int64) (bool, error) { +func hasLeaseExpired(ctx context.Context, lg *zap.Logger, cli *clientv3.Client, leaseID int64) (bool, error) { // keep retrying until lease's state is known or ctx is being canceled for ctx.Err() == nil { - resp, err := lc.getLeaseByID(ctx, leaseID) + resp, err := getLeaseByID(ctx, cli, leaseID) if err != nil { // for ~v3.1 compatibilities if rpctypes.Error(err) == rpctypes.ErrLeaseNotFound { @@ -210,9 +211,9 @@ func (lc *leaseExpireChecker) hasLeaseExpired(ctx context.Context, leaseID int64 } else { return resp.TTL == -1, nil } - lc.lg.Warn( + lg.Warn( "hasLeaseExpired getLeaseByID failed", - zap.String("endpoint", lc.m.EtcdClientEndpoint), + zap.Any("endpoint", cli.Endpoints()), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), zap.Error(err), ) @@ -223,12 +224,12 @@ func (lc *leaseExpireChecker) hasLeaseExpired(ctx context.Context, leaseID int64 // The keys attached to the lease has the format of "_" where idx is the ordering key creation // Since the format of keys contains about leaseID, finding keys base on "" prefix // determines whether the attached keys for a given leaseID has been deleted or not -func (lc *leaseExpireChecker) hasKeysAttachedToLeaseExpired(ctx context.Context, leaseID int64) (bool, error) { - resp, err := lc.cli.Get(ctx, fmt.Sprintf("%d", leaseID), clientv3.WithPrefix()) +func hasKeysAttachedToLeaseExpired(ctx context.Context, lg *zap.Logger, cli *clientv3.Client, leaseID int64) (bool, error) { + resp, err := cli.Get(ctx, fmt.Sprintf("%d", leaseID), clientv3.WithPrefix()) if err != nil { - lc.lg.Warn( + lg.Warn( "hasKeysAttachedToLeaseExpired failed", - zap.String("endpoint", lc.m.EtcdClientEndpoint), + zap.Any("endpoint", cli.Endpoints()), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), zap.Error(err), ) diff --git a/tests/functional/tester/checker_short_ttl_lease_expire.go b/tests/functional/tester/checker_short_ttl_lease_expire.go new file mode 100644 index 000000000..bf242f024 --- /dev/null +++ b/tests/functional/tester/checker_short_ttl_lease_expire.go @@ -0,0 +1,77 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "fmt" + "time" + + "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/functional/rpcpb" + + "go.uber.org/zap" + "google.golang.org/grpc" +) + +type shortTTLLeaseExpireChecker struct { + ctype rpcpb.Checker + lg *zap.Logger + m *rpcpb.Member + ls *leaseStresser + cli *clientv3.Client +} + +func newShortTTLLeaseExpireChecker(ls *leaseStresser) Checker { + return &shortTTLLeaseExpireChecker{ + ctype: rpcpb.Checker_SHORT_TTL_LEASE_EXPIRE, + lg: ls.lg, + m: ls.m, + ls: ls, + } +} + +func (lc *shortTTLLeaseExpireChecker) Type() rpcpb.Checker { + return lc.ctype +} + +func (lc *shortTTLLeaseExpireChecker) EtcdClientEndpoints() []string { + return []string{lc.m.EtcdClientEndpoint} +} + +func (lc *shortTTLLeaseExpireChecker) Check() error { + if lc.ls == nil { + return nil + } + if lc.ls != nil && lc.ls.alivedLeasesWithShortTTL == nil { + return nil + } + + cli, err := lc.m.CreateEtcdClient(grpc.WithBackoffMaxDelay(time.Second)) + if err != nil { + return fmt.Errorf("%v (%q)", err, lc.m.EtcdClientEndpoint) + } + defer func() { + if cli != nil { + cli.Close() + } + }() + lc.cli = cli + if err := check(lc.lg, lc.cli, false, lc.ls.alivedLeasesWithShortTTL.leases); err != nil { + lc.lg.Error("failed to check alivedLeasesWithShortTTL", zap.Error(err)) + return err + } + lc.lg.Info("check alivedLeasesWithShortTTL succ", zap.Int("num", len(lc.ls.alivedLeasesWithShortTTL.leases))) + return nil +} diff --git a/tests/functional/tester/cluster.go b/tests/functional/tester/cluster.go index 25d478a14..08c70afe9 100644 --- a/tests/functional/tester/cluster.go +++ b/tests/functional/tester/cluster.go @@ -327,6 +327,11 @@ func (clus *Cluster) setStresserChecker() { case "NO_CHECK": clus.checkers = append(clus.checkers, newNoChecker()) + + case "SHORT_TTL_LEASE_EXPIRE": + for _, ls := range lss { + clus.checkers = append(clus.checkers, newShortTTLLeaseExpireChecker(ls)) + } } } clus.lg.Info("updated stressers") diff --git a/tests/functional/tester/stresser_lease.go b/tests/functional/tester/stresser_lease.go index 2ce2b6aa3..a0f7cf64f 100644 --- a/tests/functional/tester/stresser_lease.go +++ b/tests/functional/tester/stresser_lease.go @@ -48,13 +48,13 @@ type leaseStresser struct { rateLimiter *rate.Limiter // atomicModifiedKey records the number of keys created and deleted during a test case - atomicModifiedKey int64 - numLeases int - keysPerLease int - - aliveLeases *atomicLeases - revokedLeases *atomicLeases - shortLivedLeases *atomicLeases + atomicModifiedKey int64 + numLeases int + keysPerLease int + aliveLeases *atomicLeases + alivedLeasesWithShortTTL *atomicLeases + revokedLeases *atomicLeases + shortLivedLeases *atomicLeases runWg sync.WaitGroup aliveWg sync.WaitGroup @@ -143,6 +143,7 @@ func (ls *leaseStresser) Stress() error { ls.revokedLeases = &atomicLeases{leases: make(map[int64]time.Time)} ls.shortLivedLeases = &atomicLeases{leases: make(map[int64]time.Time)} + ls.alivedLeasesWithShortTTL = &atomicLeases{leases: make(map[int64]time.Time)} ls.runWg.Add(1) go ls.run() @@ -193,9 +194,16 @@ func (ls *leaseStresser) restartKeepAlives() { ls.keepLeaseAlive(id) }(leaseID) } + for leaseID := range ls.alivedLeasesWithShortTTL.getLeasesMap() { + ls.aliveWg.Add(1) + go func(id int64) { + ls.keepLeaseAlive(id) + }(leaseID) + } } func (ls *leaseStresser) createLeases() { + ls.createAliveLeasesWithShortTTL() ls.createAliveLeases() ls.createShortLivedLeases() } @@ -225,6 +233,32 @@ func (ls *leaseStresser) createAliveLeases() { wg.Wait() } +func (ls *leaseStresser) createAliveLeasesWithShortTTL() { + neededLeases := 2 + var wg sync.WaitGroup + for i := 0; i < neededLeases; i++ { + wg.Add(1) + go func() { + defer wg.Done() + leaseID, err := ls.createLeaseWithKeys(defaultTTLShort) + if err != nil { + ls.lg.Debug( + "createLeaseWithKeys failed", + zap.String("endpoint", ls.m.EtcdClientEndpoint), + zap.Error(err), + ) + return + } + ls.lg.Warn("createAliveLeasesWithShortTTL", zap.Int64("lease-id", leaseID)) + ls.alivedLeasesWithShortTTL.add(leaseID, time.Now()) + // keep track of all the keep lease alive goroutines + ls.aliveWg.Add(1) + go ls.keepLeaseAlive(leaseID) + }() + } + wg.Wait() +} + func (ls *leaseStresser) createShortLivedLeases() { // one round of createLeases() might not create all the short lived leases we want due to failures. // thus, we want to create remaining short lived leases in the future round.