Mirroristas/etcd
2
0
Fork 0
mirror of https://github.com/etcd-io/etcd.git synced 2024-09-27 06:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #12898 from tangcong/add-disk-io-failure-case

Browse Source 
functional: add disk io failure case
This commit is contained in:
Gyuho Lee 2021-04-26 14:37:53 -07:00 committed by GitHub
commit c4f7d578d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 448 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tests/functional/functional.yaml
View File

@ -193,6 +193,7 @@ tester-config:
- RANDOM_DELAY_PEER_PORT_TX_RX_ALL - RANDOM_DELAY_PEER_PORT_TX_RX_ALL
- NO_FAIL_WITH_STRESS - NO_FAIL_WITH_STRESS
- NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS - NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS
# - FAILPOINTS_WITH_DISK_IO_LATENCY
# TODO: use iptables for discarding outbound rafthttp traffic to peer port # TODO: use iptables for discarding outbound rafthttp traffic to peer port
# - BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER # - BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER
@ -208,6 +209,7 @@ tester-config:
failpoint-commands: failpoint-commands:
- panic("etcd-tester") - panic("etcd-tester")
# - panic("etcd-tester"),1*sleep(1000) # - panic("etcd-tester"),1*sleep(1000)
# - sleep(3000)
runner-exec-path: ./bin/etcd-runner runner-exec-path: ./bin/etcd-runner
external-exec-path: "" external-exec-path: ""
@ -239,6 +241,7 @@ tester-config:
checkers: checkers:
- KV_HASH - KV_HASH
- LEASE_EXPIRE - LEASE_EXPIRE
#- SHORT_TTL_LEASE_EXPIRE
stress-key-size: 100 stress-key-size: 100
stress-key-size-large: 32769 stress-key-size-large: 32769

436
tests/functional/rpcpb/rpc.pb.go
View File

@ -86,10 +86,11 @@ func (StresserType) EnumDescriptor() ([]byte, []int) {
type Checker int32 type Checker int32
const ( const (
Checker_KV_HASH Checker = 0 Checker_KV_HASH Checker = 0
Checker_LEASE_EXPIRE Checker = 1 Checker_LEASE_EXPIRE Checker = 1
Checker_RUNNER Checker = 2 Checker_RUNNER Checker = 2
Checker_NO_CHECK Checker = 3 Checker_NO_CHECK Checker = 3
Checker_SHORT_TTL_LEASE_EXPIRE Checker = 4
) )
var Checker_name = map[int32]string{ var Checker_name = map[int32]string{
@ -97,13 +98,15 @@ var Checker_name = map[int32]string{
1: "LEASE_EXPIRE", 1: "LEASE_EXPIRE",
2: "RUNNER", 2: "RUNNER",
3: "NO_CHECK", 3: "NO_CHECK",
4: "SHORT_TTL_LEASE_EXPIRE",
} }
var Checker_value = map[string]int32{ var Checker_value = map[string]int32{
"KV_HASH": 0, "KV_HASH": 0,
"LEASE_EXPIRE": 1, "LEASE_EXPIRE": 1,
"RUNNER": 2, "RUNNER": 2,
"NO_CHECK": 3, "NO_CHECK": 3,
"SHORT_TTL_LEASE_EXPIRE": 4,
} }
func (x Checker) String() string { func (x Checker) String() string {
@ -507,6 +510,8 @@ const (
// FAILPOINTS injects failpoints to etcd server runtime, triggering panics // FAILPOINTS injects failpoints to etcd server runtime, triggering panics
// in critical code paths. // in critical code paths.
Case_FAILPOINTS Case = 400 Case_FAILPOINTS Case = 400
// FAILPOINTS_WITH_DISK_IO_LATENCY injects high disk I/O latency failure in raftAfterSave code paths.
Case_FAILPOINTS_WITH_DISK_IO_LATENCY Case = 401
// EXTERNAL runs external failure injection scripts. // EXTERNAL runs external failure injection scripts.
Case_EXTERNAL Case = 500 Case_EXTERNAL Case = 500
) )
@ -544,6 +549,7 @@ var Case_name = map[int32]string{
300: "NO_FAIL_WITH_STRESS", 300: "NO_FAIL_WITH_STRESS",
301: "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS", 301: "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS",
400: "FAILPOINTS", 400: "FAILPOINTS",
401: "FAILPOINTS_WITH_DISK_IO_LATENCY",
500: "EXTERNAL", 500: "EXTERNAL",
} }
@ -580,6 +586,7 @@ var Case_value = map[string]int32{
"NO_FAIL_WITH_STRESS": 300, "NO_FAIL_WITH_STRESS": 300,
"NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": 301, "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS": 301,
"FAILPOINTS": 400, "FAILPOINTS": 400,
"FAILPOINTS_WITH_DISK_IO_LATENCY": 401,
"EXTERNAL": 500, "EXTERNAL": 500,
} }
@ -1040,195 +1047,197 @@ func init() {
func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptor_4fbc93a8dcc3881e) } func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptor_4fbc93a8dcc3881e) }
var fileDescriptor_4fbc93a8dcc3881e = []byte{ var fileDescriptor_4fbc93a8dcc3881e = []byte{
// 3003 bytes of a gzipped FileDescriptorProto // 3039 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x59, 0xcb, 0x77, 0xdb, 0xc6, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x59, 0xcb, 0x77, 0xdb, 0xc6,
0xf5, 0x36, 0x44, 0x49, 0x96, 0x46, 0x2f, 0x6a, 0x64, 0xd9, 0xf0, 0x4b, 0x90, 0xe1, 0x38, 0x3f, 0xf5, 0x36, 0x44, 0x49, 0x96, 0x46, 0x2f, 0x68, 0x64, 0xd9, 0xf0, 0x4b, 0x90, 0xe1, 0x38, 0x3f,
0x59, 0x09, 0x6c, 0xff, 0xec, 0x9c, 0x3c, 0x9c, 0x26, 0x0e, 0x48, 0x41, 0x12, 0x2b, 0x88, 0xa4, 0x59, 0x09, 0x6c, 0xff, 0xec, 0x9c, 0x3c, 0x9c, 0x26, 0x0e, 0x48, 0x41, 0x12, 0x4b, 0x88, 0xa4,
0x87, 0x90, 0xec, 0x74, 0x83, 0x03, 0x91, 0x23, 0x09, 0xc7, 0x14, 0xc0, 0x00, 0x43, 0x47, 0xca, 0x87, 0x90, 0x6c, 0x77, 0x51, 0x1c, 0x88, 0x1c, 0x49, 0x3c, 0xa6, 0x00, 0x06, 0x18, 0x3a, 0x52,
0x3f, 0xd0, 0x5d, 0x4f, 0xdf, 0xa7, 0x3d, 0xa7, 0x7f, 0x42, 0xd3, 0x2e, 0xba, 0xee, 0xde, 0x79, 0xfe, 0x81, 0xee, 0x7a, 0x9a, 0x36, 0xed, 0x69, 0xcf, 0xe9, 0x9f, 0xd0, 0xb4, 0x8b, 0xae, 0xbb,
0xb5, 0x69, 0xbb, 0x6a, 0x17, 0x3c, 0x6d, 0xba, 0xe9, 0xaa, 0x0b, 0x9e, 0xbe, 0x17, 0x3d, 0x3d, 0x77, 0x5e, 0x6d, 0xda, 0xae, 0xda, 0x05, 0x4f, 0x9b, 0x6e, 0xba, 0xe6, 0xe9, 0x7b, 0xd1, 0xd3,
0x33, 0x03, 0x90, 0x03, 0x80, 0x94, 0xbd, 0xb2, 0x70, 0xef, 0xf7, 0x7d, 0x73, 0x71, 0xef, 0x60, 0x33, 0x33, 0x00, 0x39, 0x00, 0x49, 0x59, 0x2b, 0x0b, 0xf7, 0x7e, 0xdf, 0x37, 0x17, 0xf7, 0x0e,
0xee, 0x1d, 0x13, 0xcc, 0x05, 0xad, 0x7a, 0x6b, 0xef, 0x76, 0xd0, 0xaa, 0xdf, 0x6a, 0x05, 0x3e, 0xe6, 0xde, 0x31, 0xc1, 0x5c, 0xd0, 0xac, 0x36, 0x77, 0x6f, 0x07, 0xcd, 0xea, 0xad, 0x66, 0xe0,
0xf1, 0xe1, 0x18, 0x33, 0x5c, 0xd2, 0x0e, 0x5c, 0x72, 0xd8, 0xde, 0xbb, 0x55, 0xf7, 0x8f, 0x6e, 0x13, 0x1f, 0x8e, 0x31, 0xc3, 0x25, 0x7d, 0xbf, 0x4e, 0x0e, 0x5a, 0xbb, 0xb7, 0xaa, 0xfe, 0xe1,
0x1f, 0xf8, 0x07, 0xfe, 0x6d, 0xe6, 0xdd, 0x6b, 0xef, 0xb3, 0x27, 0xf6, 0xc0, 0xfe, 0xe2, 0x2c, 0xed, 0x7d, 0x7f, 0xdf, 0xbf, 0xcd, 0xbc, 0xbb, 0xad, 0x3d, 0xf6, 0xc4, 0x1e, 0xd8, 0x5f, 0x9c,
0xf5, 0x9b, 0x12, 0x38, 0x8b, 0xf0, 0x07, 0x6d, 0x1c, 0x12, 0x78, 0x0b, 0x4c, 0x56, 0x5a, 0x38, 0xa5, 0x7d, 0x47, 0x02, 0x67, 0x11, 0x7e, 0xbf, 0x85, 0x43, 0x02, 0x6f, 0x81, 0xc9, 0x52, 0x13,
0x70, 0x88, 0xeb, 0x7b, 0xb2, 0xb4, 0x2c, 0xad, 0xcc, 0xde, 0xcd, 0xdf, 0x62, 0xaa, 0xb7, 0x7a, 0x07, 0x2e, 0xa9, 0xfb, 0x9e, 0x22, 0x2d, 0x4b, 0x2b, 0xb3, 0x77, 0xe5, 0x5b, 0x4c, 0xf5, 0x56,
0x76, 0xd4, 0x87, 0xc0, 0x1b, 0x60, 0x7c, 0x1b, 0x1f, 0xed, 0xe1, 0x40, 0x1e, 0x59, 0x96, 0x56, 0xd7, 0x8e, 0x7a, 0x10, 0x78, 0x03, 0x8c, 0x6f, 0xe1, 0xc3, 0x5d, 0x1c, 0x28, 0x23, 0xcb, 0xd2,
0xa6, 0xee, 0xce, 0x44, 0x60, 0x6e, 0x44, 0x91, 0x93, 0xc2, 0x2c, 0x1c, 0x12, 0x1c, 0xc8, 0xb9, 0xca, 0xd4, 0xdd, 0x99, 0x08, 0xcc, 0x8d, 0x28, 0x72, 0x52, 0x98, 0x8d, 0x43, 0x82, 0x03, 0x25,
0x04, 0x8c, 0x1b, 0x51, 0xe4, 0x54, 0xff, 0x32, 0x02, 0xa6, 0x6b, 0x9e, 0xd3, 0x0a, 0x0f, 0x7d, 0x93, 0x80, 0x71, 0x23, 0x8a, 0x9c, 0xda, 0x5f, 0x47, 0xc0, 0x74, 0xc5, 0x73, 0x9b, 0xe1, 0x81,
0x52, 0xf2, 0xf6, 0x7d, 0xb8, 0x04, 0x00, 0x57, 0x28, 0x3b, 0x47, 0x98, 0xc5, 0x33, 0x89, 0x04, 0x4f, 0xf2, 0xde, 0x9e, 0x0f, 0x97, 0x00, 0xe0, 0x0a, 0x45, 0xf7, 0x10, 0xb3, 0x78, 0x26, 0x91,
0x0b, 0x5c, 0x05, 0x79, 0xfe, 0x54, 0x6c, 0xba, 0xd8, 0x23, 0x3b, 0xc8, 0x0c, 0xe5, 0x91, 0xe5, 0x60, 0x81, 0xab, 0x40, 0xe6, 0x4f, 0xb9, 0x46, 0x1d, 0x7b, 0x64, 0x1b, 0x59, 0xa1, 0x32, 0xb2,
0xdc, 0xca, 0x24, 0xca, 0xd8, 0xa1, 0xda, 0xd7, 0xae, 0x3a, 0xe4, 0x90, 0x45, 0x32, 0x89, 0x12, 0x9c, 0x59, 0x99, 0x44, 0x7d, 0x76, 0xa8, 0xf5, 0xb4, 0xcb, 0x2e, 0x39, 0x60, 0x91, 0x4c, 0xa2,
0x36, 0xaa, 0x17, 0x3f, 0xaf, 0xbb, 0x4d, 0x5c, 0x73, 0x3f, 0xc2, 0xf2, 0x28, 0xc3, 0x65, 0xec, 0x84, 0x8d, 0xea, 0xc5, 0xcf, 0xeb, 0xf5, 0x06, 0xae, 0xd4, 0x3f, 0xc4, 0xca, 0x28, 0xc3, 0xf5,
0xf0, 0x55, 0x30, 0x1f, 0xdb, 0x2c, 0x9f, 0x38, 0x4d, 0x06, 0x1e, 0x63, 0xe0, 0xac, 0x43, 0x54, 0xd9, 0xe1, 0xab, 0x60, 0x3e, 0xb6, 0xd9, 0x3e, 0x71, 0x1b, 0x0c, 0x3c, 0xc6, 0xc0, 0xfd, 0x0e,
0x66, 0xc6, 0x2d, 0x7c, 0x22, 0x8f, 0x2f, 0x4b, 0x2b, 0x39, 0x94, 0xb1, 0x8b, 0x91, 0x6e, 0x3a, 0x51, 0x99, 0x19, 0x0b, 0xf8, 0x58, 0x19, 0x5f, 0x96, 0x56, 0x32, 0xa8, 0xcf, 0x2e, 0x46, 0xba,
0xe1, 0xa1, 0x7c, 0x96, 0xe1, 0x12, 0x36, 0x51, 0x0f, 0xe1, 0xa7, 0x6e, 0x48, 0xeb, 0x35, 0x91, 0xe9, 0x86, 0x07, 0xca, 0x59, 0x86, 0x4b, 0xd8, 0x44, 0x3d, 0x84, 0x9f, 0xd5, 0x43, 0x5a, 0xaf,
0xd4, 0x8b, 0xed, 0x10, 0x82, 0x51, 0xcb, 0xf7, 0x9f, 0xc8, 0x93, 0x2c, 0x38, 0xf6, 0xb7, 0xfa, 0x89, 0xa4, 0x5e, 0x6c, 0x87, 0x10, 0x8c, 0xda, 0xbe, 0xff, 0x54, 0x99, 0x64, 0xc1, 0xb1, 0xbf,
0x13, 0x09, 0x4c, 0x20, 0x1c, 0xb6, 0x7c, 0x2f, 0xc4, 0x50, 0x06, 0x67, 0x6b, 0xed, 0x7a, 0x1d, 0xb5, 0x9f, 0x4a, 0x60, 0x02, 0xe1, 0xb0, 0xe9, 0x7b, 0x21, 0x86, 0x0a, 0x38, 0x5b, 0x69, 0x55,
0x87, 0x21, 0xcb, 0xf1, 0x04, 0x8a, 0x1f, 0xe1, 0x79, 0x30, 0x5e, 0x23, 0x0e, 0x69, 0x87, 0xac, 0xab, 0x38, 0x0c, 0x59, 0x8e, 0x27, 0x50, 0xfc, 0x08, 0xcf, 0x83, 0xf1, 0x0a, 0x71, 0x49, 0x2b,
0xbe, 0x93, 0x28, 0x7a, 0x12, 0xea, 0x9e, 0x3b, 0xad, 0xee, 0x6f, 0x24, 0xeb, 0xc9, 0x72, 0x39, 0x64, 0xf5, 0x9d, 0x44, 0xd1, 0x93, 0x50, 0xf7, 0xcc, 0x49, 0x75, 0x7f, 0x23, 0x59, 0x4f, 0x96,
0x75, 0x77, 0x21, 0x02, 0x8b, 0x2e, 0x94, 0x00, 0xaa, 0x9f, 0x4e, 0xc7, 0x0b, 0xc0, 0x3b, 0x60, 0xcb, 0xa9, 0xbb, 0x0b, 0x11, 0x58, 0x74, 0xa1, 0x04, 0x50, 0xfb, 0x6c, 0x3a, 0x5e, 0x00, 0xde,
0xc2, 0x20, 0xf5, 0x86, 0x71, 0x8c, 0xeb, 0x7c, 0x07, 0x14, 0xce, 0x75, 0x3b, 0x4a, 0xfe, 0xc4, 0x01, 0x13, 0x26, 0xa9, 0xd6, 0xcc, 0x23, 0x5c, 0xe5, 0x3b, 0x20, 0x7b, 0xae, 0xd3, 0x56, 0xe5,
0x39, 0x6a, 0xde, 0x57, 0x31, 0xa9, 0x37, 0x34, 0x7c, 0x8c, 0xeb, 0x2a, 0xea, 0xa1, 0xe0, 0x3d, 0x63, 0xf7, 0xb0, 0x71, 0x5f, 0xc3, 0xa4, 0x5a, 0xd3, 0xf1, 0x11, 0xae, 0x6a, 0xa8, 0x8b, 0x82,
0x30, 0xa9, 0x1f, 0x60, 0x8f, 0xe8, 0x8d, 0x46, 0x20, 0x4f, 0x31, 0xca, 0x62, 0xb7, 0xa3, 0xcc, 0xf7, 0xc0, 0xa4, 0xb1, 0x8f, 0x3d, 0x62, 0xd4, 0x6a, 0x81, 0x32, 0xc5, 0x28, 0x8b, 0x9d, 0xb6,
0x73, 0x8a, 0x43, 0x5d, 0x9a, 0xd3, 0x68, 0x04, 0x2a, 0xea, 0xe3, 0xa0, 0x09, 0xe6, 0xd7, 0x1d, 0x3a, 0xcf, 0x29, 0x2e, 0x75, 0xe9, 0x6e, 0xad, 0x16, 0x68, 0xa8, 0x87, 0x83, 0x16, 0x98, 0x5f,
0xb7, 0xd9, 0xf2, 0x5d, 0x8f, 0x6c, 0x5a, 0x56, 0x95, 0x91, 0xa7, 0x19, 0x79, 0xa9, 0xdb, 0x51, 0x77, 0xeb, 0x8d, 0xa6, 0x5f, 0xf7, 0xc8, 0xa6, 0x6d, 0x97, 0x19, 0x79, 0x9a, 0x91, 0x97, 0x3a,
0x2e, 0x71, 0xf2, 0x7e, 0x0c, 0xd1, 0x0e, 0x09, 0x69, 0x45, 0x2a, 0x59, 0x22, 0xd4, 0xc0, 0xd9, 0x6d, 0xf5, 0x12, 0x27, 0xef, 0xc5, 0x10, 0xfd, 0x80, 0x90, 0x66, 0xa4, 0xd2, 0x4f, 0x84, 0x3a,
0x82, 0x13, 0xe2, 0x35, 0x37, 0x90, 0x31, 0xd3, 0x58, 0xe8, 0x76, 0x94, 0x39, 0xae, 0xb1, 0xe7, 0x38, 0x9b, 0x75, 0x43, 0xbc, 0x56, 0x0f, 0x14, 0xcc, 0x34, 0x16, 0x3a, 0x6d, 0x75, 0x8e, 0x6b,
0x84, 0x58, 0x6b, 0xb8, 0x81, 0x8a, 0x62, 0x0c, 0xdc, 0x00, 0x73, 0x34, 0x7a, 0xbe, 0x5b, 0xab, 0xec, 0xba, 0x21, 0xd6, 0x6b, 0xf5, 0x40, 0x43, 0x31, 0x06, 0x6e, 0x80, 0x39, 0x1a, 0x3d, 0xdf,
0x81, 0x7f, 0x7c, 0x22, 0x7f, 0xc2, 0x2a, 0x51, 0xb8, 0xd2, 0xed, 0x28, 0xb2, 0xf0, 0xae, 0x75, 0xad, 0xe5, 0xc0, 0x3f, 0x3a, 0x56, 0x3e, 0x65, 0x95, 0xc8, 0x5e, 0xe9, 0xb4, 0x55, 0x45, 0x78,
0x06, 0xd1, 0x5a, 0x14, 0xa3, 0xa2, 0x34, 0x0b, 0xea, 0x60, 0x86, 0x9a, 0xaa, 0x18, 0x07, 0x5c, 0xd7, 0x2a, 0x83, 0xe8, 0x4d, 0x8a, 0xd1, 0x50, 0x9a, 0x05, 0x0d, 0x30, 0x43, 0x4d, 0x65, 0x8c,
0xe6, 0x53, 0x2e, 0x73, 0xa9, 0xdb, 0x51, 0xce, 0x0b, 0x32, 0x2d, 0x8c, 0x83, 0x58, 0x24, 0xc9, 0x03, 0x2e, 0xf3, 0x19, 0x97, 0xb9, 0xd4, 0x69, 0xab, 0xe7, 0x05, 0x99, 0x26, 0xc6, 0x41, 0x2c,
0x80, 0x55, 0x00, 0xfb, 0xaa, 0x86, 0xd7, 0x60, 0x2f, 0x26, 0x7f, 0xcc, 0xea, 0x5f, 0x50, 0xba, 0x92, 0x64, 0xc0, 0x32, 0x80, 0x3d, 0x55, 0xd3, 0xab, 0xb1, 0x17, 0x53, 0x3e, 0x61, 0xf5, 0xcf,
0x1d, 0xe5, 0x72, 0x36, 0x1c, 0x1c, 0xc1, 0x54, 0x34, 0x80, 0x0b, 0xff, 0x1f, 0x8c, 0x52, 0xab, 0xaa, 0x9d, 0xb6, 0x7a, 0xb9, 0x3f, 0x1c, 0x1c, 0xc1, 0x34, 0x34, 0x80, 0x0b, 0xff, 0x1f, 0x8c,
0xfc, 0x33, 0x7e, 0x46, 0x4c, 0x45, 0xe5, 0xa7, 0xb6, 0xc2, 0x5c, 0xb7, 0xa3, 0x4c, 0xf5, 0x05, 0x52, 0xab, 0xf2, 0x73, 0x7e, 0x46, 0x4c, 0x45, 0xe5, 0xa7, 0xb6, 0xec, 0x5c, 0xa7, 0xad, 0x4e,
0x55, 0xc4, 0xa0, 0xb0, 0x00, 0x16, 0xe9, 0xbf, 0x15, 0xaf, 0xbf, 0x99, 0x43, 0xe2, 0x07, 0x58, 0xf5, 0x04, 0x35, 0xc4, 0xa0, 0x30, 0x0b, 0x16, 0xe9, 0xbf, 0x25, 0xaf, 0xb7, 0x99, 0x43, 0xe2,
0xfe, 0x79, 0x56, 0x03, 0x0d, 0x86, 0xc2, 0x35, 0x30, 0xcb, 0x03, 0x29, 0xe2, 0x80, 0xac, 0x39, 0x07, 0x58, 0xf9, 0x45, 0xbf, 0x06, 0x1a, 0x0c, 0x85, 0x6b, 0x60, 0x96, 0x07, 0x92, 0xc3, 0x01,
0xc4, 0x91, 0xbf, 0xc3, 0xbe, 0xf9, 0xc2, 0xe5, 0x6e, 0x47, 0xb9, 0xc0, 0xd7, 0x8c, 0xe2, 0xaf, 0x59, 0x73, 0x89, 0xab, 0x7c, 0xc4, 0xbe, 0xf9, 0xec, 0xe5, 0x4e, 0x5b, 0xbd, 0xc0, 0xd7, 0x8c,
0xe3, 0x80, 0x68, 0x0d, 0x87, 0x38, 0x2a, 0x4a, 0x71, 0x92, 0x2a, 0xec, 0xe0, 0xf8, 0xee, 0xa9, 0xe2, 0xaf, 0xe2, 0x80, 0xe8, 0x35, 0x97, 0xb8, 0x1a, 0x4a, 0x71, 0x92, 0x2a, 0xec, 0xe0, 0xf8,
0x2a, 0x2d, 0x87, 0x1c, 0x26, 0x54, 0xd8, 0xc1, 0xa2, 0x83, 0x19, 0x6e, 0xd9, 0xc2, 0x27, 0x2c, 0xfe, 0x89, 0x2a, 0x4d, 0x97, 0x1c, 0x24, 0x54, 0xd8, 0xc1, 0x62, 0x80, 0x19, 0x6e, 0x29, 0xe0,
0x94, 0xef, 0x71, 0x11, 0xa1, 0x2e, 0x91, 0xc8, 0x13, 0x7c, 0x12, 0x45, 0x92, 0x64, 0x24, 0x24, 0x63, 0x16, 0xca, 0x0f, 0xb8, 0x88, 0x50, 0x97, 0x48, 0xe4, 0x29, 0x3e, 0x8e, 0x22, 0x49, 0x32,
0x58, 0x1c, 0xdf, 0x3f, 0x4d, 0x82, 0x87, 0x91, 0x64, 0x40, 0x0b, 0x2c, 0x70, 0x83, 0x15, 0xb4, 0x12, 0x12, 0x2c, 0x8e, 0x8f, 0x4f, 0x92, 0xe0, 0x61, 0x24, 0x19, 0xd0, 0x06, 0x0b, 0xdc, 0x60,
0x43, 0x82, 0x1b, 0x45, 0x9d, 0xc5, 0xf2, 0x03, 0x2e, 0x74, 0xad, 0xdb, 0x51, 0xae, 0x26, 0x84, 0x07, 0xad, 0x90, 0xe0, 0x5a, 0xce, 0x60, 0xb1, 0xfc, 0x90, 0x0b, 0x5d, 0xeb, 0xb4, 0xd5, 0xab,
0x08, 0x87, 0x69, 0x75, 0x27, 0x0a, 0x69, 0x10, 0x7d, 0x80, 0x2a, 0x0b, 0xef, 0x87, 0x2f, 0xa0, 0x09, 0x21, 0xc2, 0x61, 0x7a, 0xd5, 0x8d, 0x42, 0x1a, 0x44, 0x1f, 0xa0, 0xca, 0xc2, 0xfb, 0xd1,
0xca, 0xa3, 0x1c, 0x44, 0x87, 0xef, 0x82, 0x69, 0xba, 0x27, 0x7b, 0xb5, 0xfb, 0x3b, 0x97, 0xbb, 0x29, 0x54, 0x79, 0x94, 0x83, 0xe8, 0xf0, 0x5d, 0x30, 0x4d, 0xf7, 0x64, 0xb7, 0x76, 0x7f, 0xe7,
0xd8, 0xed, 0x28, 0x8b, 0x5c, 0x8e, 0xed, 0x61, 0xa1, 0x72, 0x09, 0xbc, 0xc8, 0x67, 0xe1, 0xfc, 0x72, 0x17, 0x3b, 0x6d, 0x75, 0x91, 0xcb, 0xb1, 0x3d, 0x2c, 0x54, 0x2e, 0x81, 0x17, 0xf9, 0x2c,
0xe3, 0x14, 0x3e, 0x0f, 0x23, 0x81, 0x87, 0x6f, 0x83, 0x29, 0xfa, 0x1c, 0xd7, 0xeb, 0x9f, 0x9c, 0x9c, 0x7f, 0x9c, 0xc0, 0xe7, 0x61, 0x24, 0xf0, 0xf0, 0x6d, 0x30, 0x45, 0x9f, 0xe3, 0x7a, 0xfd,
0x2e, 0x77, 0x3b, 0xca, 0x39, 0x81, 0xde, 0xaf, 0x96, 0x88, 0x16, 0xc8, 0x6c, 0xed, 0x7f, 0x0d, 0x93, 0xd3, 0x95, 0x4e, 0x5b, 0x3d, 0x27, 0xd0, 0x7b, 0xd5, 0x12, 0xd1, 0x02, 0x99, 0xad, 0xfd,
0x27, 0xf3, 0xa5, 0x45, 0x34, 0x2c, 0x83, 0x79, 0xfa, 0x98, 0xac, 0xd1, 0xbf, 0x73, 0xe9, 0xef, 0xaf, 0xe1, 0x64, 0xbe, 0xb4, 0x88, 0x86, 0x45, 0x30, 0x4f, 0x1f, 0x93, 0x35, 0xfa, 0x77, 0x26,
0x8f, 0x49, 0x64, 0x2a, 0x94, 0xa5, 0x66, 0xf4, 0x58, 0x48, 0xff, 0x79, 0xae, 0x1e, 0x8f, 0x2c, 0xfd, 0xfd, 0x31, 0x89, 0xbe, 0x0a, 0xf5, 0x53, 0xfb, 0xf4, 0x58, 0x48, 0xff, 0x79, 0xa1, 0x1e,
0x4b, 0x85, 0xef, 0xa4, 0x1a, 0xe9, 0xef, 0x47, 0xd3, 0x6f, 0x17, 0x46, 0xee, 0x38, 0xb1, 0x89, 0x8f, 0xac, 0x9f, 0x0a, 0xdf, 0x49, 0x35, 0xd2, 0x3f, 0x8c, 0xa6, 0xdf, 0x2e, 0x8c, 0xdc, 0x71,
0x1e, 0xfb, 0x66, 0xaa, 0x27, 0xfc, 0xe1, 0x45, 0x9b, 0x02, 0x7c, 0x1d, 0x80, 0xde, 0x49, 0x1b, 0x62, 0x13, 0x3d, 0xf6, 0xcd, 0x54, 0x4f, 0xf8, 0xe3, 0x69, 0x9b, 0x02, 0x7c, 0x1d, 0x80, 0xee,
0xca, 0xbf, 0x1c, 0x4b, 0x9f, 0xec, 0xbd, 0xc3, 0x39, 0x54, 0x91, 0x80, 0x54, 0x7f, 0x31, 0x1d, 0x49, 0x1b, 0x2a, 0xbf, 0x1a, 0x4b, 0x9f, 0xec, 0xdd, 0xc3, 0x39, 0xd4, 0x90, 0x80, 0xd4, 0x7e,
0x8f, 0x1f, 0xf4, 0x5c, 0xa6, 0x39, 0xa1, 0xe7, 0xb2, 0x94, 0x3e, 0x97, 0x69, 0x02, 0xa3, 0x73, 0x39, 0x1d, 0x8f, 0x1f, 0xf4, 0x5c, 0xa6, 0x39, 0xa1, 0xe7, 0xb2, 0x94, 0x3e, 0x97, 0x69, 0x02,
0x39, 0xc2, 0xc0, 0x57, 0xc1, 0xd9, 0x32, 0x26, 0x1f, 0xfa, 0xc1, 0x13, 0xde, 0xff, 0x0a, 0xb0, 0xa3, 0x73, 0x39, 0xc2, 0xc0, 0x57, 0xc1, 0xd9, 0x22, 0x26, 0x1f, 0xf8, 0xc1, 0x53, 0xde, 0xff,
0xdb, 0x51, 0x66, 0x39, 0xdc, 0xe3, 0x0e, 0x15, 0xc5, 0x10, 0x78, 0x1d, 0x8c, 0xb2, 0xae, 0xc1, 0xb2, 0xb0, 0xd3, 0x56, 0x67, 0x39, 0xdc, 0xe3, 0x0e, 0x0d, 0xc5, 0x10, 0x78, 0x1d, 0x8c, 0xb2,
0x53, 0x2b, 0x9c, 0x6c, 0xbc, 0x4d, 0x30, 0x27, 0x2c, 0x82, 0xd9, 0x35, 0xdc, 0x74, 0x4e, 0x4c, 0xae, 0xc1, 0x53, 0x2b, 0x9c, 0x6c, 0xbc, 0x4d, 0x30, 0x27, 0xcc, 0x81, 0xd9, 0x35, 0xdc, 0x70,
0x87, 0x60, 0xaf, 0x7e, 0xb2, 0x1d, 0xb2, 0x0e, 0x35, 0x23, 0x1e, 0x27, 0x0d, 0xea, 0xd7, 0x9a, 0x8f, 0x2d, 0x97, 0x60, 0xaf, 0x7a, 0xbc, 0x15, 0xb2, 0x0e, 0x35, 0x23, 0x1e, 0x27, 0x35, 0xea,
0x1c, 0xa0, 0x1d, 0x85, 0x2a, 0x4a, 0x51, 0xe0, 0xd7, 0x41, 0x3e, 0x69, 0x41, 0x4f, 0x59, 0xaf, 0xd7, 0x1b, 0x1c, 0xa0, 0x1f, 0x86, 0x1a, 0x4a, 0x51, 0xe0, 0x37, 0x81, 0x9c, 0xb4, 0xa0, 0x67,
0x9a, 0x11, 0x7b, 0x55, 0x5a, 0x46, 0x0b, 0x9e, 0xaa, 0x28, 0xc3, 0x83, 0xef, 0x83, 0xc5, 0x9d, 0xac, 0x57, 0xcd, 0x88, 0xbd, 0x2a, 0x2d, 0xa3, 0x07, 0xcf, 0x34, 0xd4, 0xc7, 0x83, 0x4f, 0xc0,
0x56, 0xc3, 0x21, 0xb8, 0x91, 0x8a, 0x6b, 0x86, 0x09, 0x5e, 0xef, 0x76, 0x14, 0x85, 0x0b, 0xb6, 0xe2, 0x76, 0xb3, 0xe6, 0x12, 0x5c, 0x4b, 0xc5, 0x35, 0xc3, 0x04, 0xaf, 0x77, 0xda, 0xaa, 0xca,
0x39, 0x4c, 0xcb, 0xc6, 0x37, 0x58, 0x81, 0x16, 0x0c, 0xf9, 0x6d, 0xaf, 0x61, 0xba, 0x47, 0x2e, 0x05, 0x5b, 0x1c, 0xa6, 0xf7, 0xc7, 0x37, 0x58, 0x81, 0x16, 0x0c, 0xf9, 0x2d, 0xaf, 0x66, 0xd5,
0x91, 0x17, 0x97, 0xa5, 0x95, 0xb1, 0xc2, 0xf9, 0x6e, 0x47, 0x81, 0x5c, 0x2f, 0xa0, 0x3e, 0xad, 0x0f, 0xeb, 0x44, 0x59, 0x5c, 0x96, 0x56, 0xc6, 0xb2, 0xe7, 0x3b, 0x6d, 0x15, 0x72, 0xbd, 0x80,
0x49, 0x9d, 0x2a, 0x12, 0x90, 0xb0, 0x00, 0x66, 0x8d, 0x63, 0x97, 0x54, 0xbc, 0xa2, 0x13, 0x62, 0xfa, 0xf4, 0x06, 0x75, 0x6a, 0x48, 0x40, 0xc2, 0x2c, 0x98, 0x35, 0x8f, 0xea, 0xa4, 0xe4, 0xe5,
0x5a, 0x48, 0xf9, 0x7c, 0xa6, 0x8b, 0x1d, 0xbb, 0x44, 0xf3, 0x3d, 0x8d, 0xd6, 0xbc, 0x1d, 0x60, 0xdc, 0x10, 0xd3, 0x42, 0x2a, 0xe7, 0xfb, 0xba, 0xd8, 0x51, 0x9d, 0xe8, 0xbe, 0xa7, 0xd3, 0x9a,
0x15, 0xa5, 0x18, 0xf0, 0x2d, 0x30, 0x65, 0x78, 0xce, 0x5e, 0x13, 0x57, 0x5b, 0x81, 0xbf, 0x2f, 0xb7, 0x02, 0xac, 0xa1, 0x14, 0x03, 0xbe, 0x05, 0xa6, 0x4c, 0xcf, 0xdd, 0x6d, 0xe0, 0x72, 0x33,
0x5f, 0x60, 0x02, 0x17, 0xba, 0x1d, 0x65, 0x21, 0x12, 0x60, 0x4e, 0xad, 0x45, 0xbd, 0x2a, 0x12, 0xf0, 0xf7, 0x94, 0x0b, 0x4c, 0xe0, 0x42, 0xa7, 0xad, 0x2e, 0x44, 0x02, 0xcc, 0xa9, 0x37, 0xa9,
0xb1, 0xf0, 0x3e, 0x98, 0xa2, 0x32, 0xec, 0x65, 0xb6, 0x43, 0x59, 0x61, 0x79, 0x10, 0xb6, 0x77, 0x57, 0x43, 0x22, 0x16, 0xde, 0x07, 0x53, 0x54, 0x86, 0xbd, 0xcc, 0x56, 0xa8, 0xa8, 0x2c, 0x0f,
0x9d, 0x35, 0x70, 0x96, 0x04, 0xfa, 0xf2, 0x22, 0x98, 0x2e, 0x4b, 0x1f, 0x6b, 0x87, 0xed, 0xfd, 0xc2, 0xf6, 0xae, 0xb2, 0x06, 0xce, 0x92, 0x40, 0x5f, 0x5e, 0x04, 0xd3, 0x65, 0xe9, 0x63, 0xe5,
0xfd, 0x26, 0x96, 0x97, 0xd3, 0xcb, 0x32, 0x6e, 0xc8, 0xbd, 0x11, 0x35, 0xc2, 0xc2, 0x97, 0xc1, 0xa0, 0xb5, 0xb7, 0xd7, 0xc0, 0xca, 0x72, 0x7a, 0x59, 0xc6, 0x0d, 0xb9, 0x37, 0xa2, 0x46, 0x58,
0x18, 0x7d, 0x0c, 0xe5, 0x6b, 0x74, 0x82, 0x2d, 0xe4, 0xbb, 0x1d, 0x65, 0xba, 0x4f, 0x0a, 0x55, 0xf8, 0x32, 0x18, 0xa3, 0x8f, 0xa1, 0x72, 0x8d, 0x4e, 0xb0, 0x59, 0xb9, 0xd3, 0x56, 0xa7, 0x7b,
0xc4, 0xdd, 0x70, 0x4b, 0x98, 0x54, 0x8a, 0xfe, 0xd1, 0x91, 0xe3, 0x35, 0x42, 0x59, 0x65, 0x9c, 0xa4, 0x50, 0x43, 0xdc, 0x0d, 0x0b, 0xc2, 0xa4, 0x92, 0xf3, 0x0f, 0x0f, 0x5d, 0xaf, 0x16, 0x2a,
0xab, 0xdd, 0x8e, 0x72, 0x31, 0x3d, 0xa9, 0xd4, 0x23, 0x8c, 0x38, 0xa8, 0xc4, 0x3c, 0xba, 0x1d, 0x1a, 0xe3, 0x5c, 0xed, 0xb4, 0xd5, 0x8b, 0xe9, 0x49, 0xa5, 0x1a, 0x61, 0xc4, 0x41, 0x25, 0xe6,
0x51, 0xdb, 0xf3, 0x70, 0x40, 0x27, 0x27, 0xf6, 0x39, 0xdf, 0x4c, 0x77, 0xb7, 0x80, 0xf9, 0xd9, 0xd1, 0xed, 0x88, 0x5a, 0x9e, 0x87, 0x03, 0x3a, 0x39, 0xb1, 0xcf, 0xf9, 0x66, 0xba, 0xbb, 0x05,
0x94, 0x15, 0x77, 0xb7, 0x24, 0x05, 0x96, 0x40, 0xde, 0x38, 0x26, 0x38, 0xf0, 0x9c, 0x66, 0x4f, 0xcc, 0xcf, 0xa6, 0xac, 0xb8, 0xbb, 0x25, 0x29, 0x30, 0x0f, 0x64, 0xf3, 0x88, 0xe0, 0xc0, 0x73,
0x66, 0x95, 0xc9, 0x08, 0x01, 0xe1, 0x08, 0x21, 0x0a, 0x65, 0x68, 0xb0, 0x08, 0x26, 0x6b, 0x24, 0x1b, 0x5d, 0x99, 0x55, 0x26, 0x23, 0x04, 0x84, 0x23, 0x84, 0x28, 0xd4, 0x47, 0x83, 0x39, 0x30,
0xc0, 0x61, 0x88, 0x83, 0x50, 0xc6, 0xcb, 0xb9, 0x95, 0xa9, 0xbb, 0x73, 0xf1, 0xc9, 0x10, 0xd9, 0x59, 0x21, 0x01, 0x0e, 0x43, 0x1c, 0x84, 0x0a, 0x5e, 0xce, 0xac, 0x4c, 0xdd, 0x9d, 0x8b, 0x4f,
0xc5, 0xf9, 0x2f, 0x8c, 0xb1, 0x2a, 0xea, 0xf3, 0xe0, 0x6d, 0x30, 0x51, 0x3c, 0xc4, 0xf5, 0x27, 0x86, 0xc8, 0x2e, 0xce, 0x7f, 0x61, 0x8c, 0xd5, 0x50, 0x8f, 0x07, 0x6f, 0x83, 0x89, 0xdc, 0x01,
0x54, 0x63, 0x9f, 0x25, 0x46, 0xf8, 0xcc, 0xeb, 0x91, 0x47, 0x45, 0x3d, 0x10, 0xed, 0xad, 0x9c, 0xae, 0x3e, 0xa5, 0x1a, 0x7b, 0x2c, 0x31, 0xc2, 0x67, 0x5e, 0x8d, 0x3c, 0x1a, 0xea, 0x82, 0x68,
0xbd, 0x85, 0x4f, 0xd8, 0x1c, 0xcf, 0xa6, 0xaf, 0x31, 0x71, 0xc3, 0xf1, 0x95, 0xd8, 0x99, 0x1d, 0x6f, 0xe5, 0xec, 0x02, 0x3e, 0x66, 0x73, 0x3c, 0x9b, 0xbe, 0xc6, 0xc4, 0x0d, 0xc7, 0x57, 0x62,
0xba, 0x1f, 0x61, 0x15, 0x25, 0x19, 0xf0, 0x21, 0x80, 0x09, 0x83, 0xe9, 0x04, 0x07, 0x98, 0x8f, 0x67, 0x76, 0x58, 0xff, 0x10, 0x6b, 0x28, 0xc9, 0x80, 0x0f, 0x01, 0x4c, 0x18, 0x2c, 0x37, 0xd8,
0x5f, 0x63, 0x85, 0xe5, 0x6e, 0x47, 0xb9, 0x32, 0x50, 0x47, 0x6b, 0x52, 0x9c, 0x8a, 0x06, 0x90, 0xc7, 0x7c, 0xfc, 0x1a, 0xcb, 0x2e, 0x77, 0xda, 0xea, 0x95, 0x81, 0x3a, 0x7a, 0x83, 0xe2, 0x34,
0xe1, 0x23, 0x70, 0xae, 0x6f, 0x6d, 0xef, 0xef, 0xbb, 0xc7, 0xc8, 0xf1, 0x0e, 0xb0, 0xfc, 0x19, 0x34, 0x80, 0x0c, 0x1f, 0x81, 0x73, 0x3d, 0x6b, 0x6b, 0x6f, 0xaf, 0x7e, 0x84, 0x5c, 0x6f, 0x1f,
0x17, 0x55, 0xbb, 0x1d, 0x65, 0x29, 0x2b, 0xca, 0x80, 0x5a, 0x40, 0x91, 0x2a, 0x1a, 0x28, 0x00, 0x2b, 0x9f, 0x73, 0x51, 0xad, 0xd3, 0x56, 0x97, 0xfa, 0x45, 0x19, 0x50, 0x0f, 0x28, 0x52, 0x43,
0x1d, 0x70, 0x61, 0x90, 0xdd, 0x3a, 0xf6, 0xe4, 0xcf, 0xb9, 0xf6, 0xcb, 0xdd, 0x8e, 0xa2, 0x9e, 0x03, 0x05, 0xa0, 0x0b, 0x2e, 0x0c, 0xb2, 0xdb, 0x47, 0x9e, 0xf2, 0x05, 0xd7, 0x7e, 0xb9, 0xd3,
0xaa, 0xad, 0x91, 0x63, 0x4f, 0x45, 0xc3, 0x74, 0xe0, 0x26, 0x98, 0xeb, 0xb9, 0xac, 0x63, 0xaf, 0x56, 0xb5, 0x13, 0xb5, 0x75, 0x72, 0xe4, 0x69, 0x68, 0x98, 0x0e, 0xdc, 0x04, 0x73, 0x5d, 0x97,
0xd2, 0x0a, 0xe5, 0x2f, 0xb8, 0xb4, 0xb0, 0x25, 0x04, 0x69, 0x72, 0xec, 0x69, 0x7e, 0x2b, 0x54, 0x7d, 0xe4, 0x95, 0x9a, 0xa1, 0xf2, 0x25, 0x97, 0x16, 0xb6, 0x84, 0x20, 0x4d, 0x8e, 0x3c, 0xdd,
0x51, 0x9a, 0x06, 0xdf, 0x8b, 0x6b, 0xc3, 0xa7, 0x84, 0x90, 0x8f, 0xa2, 0x63, 0x62, 0x27, 0x8f, 0x6f, 0x86, 0x1a, 0x4a, 0xd3, 0xe0, 0x7b, 0x71, 0x6d, 0xf8, 0x94, 0x10, 0xf2, 0x51, 0x74, 0x4c,
0x74, 0xf8, 0x7c, 0x11, 0xf6, 0x4a, 0x13, 0x11, 0xe0, 0x6b, 0xf1, 0x9e, 0x7a, 0x58, 0xad, 0xf1, 0xec, 0xe4, 0x91, 0x0e, 0x9f, 0x2f, 0xc2, 0x6e, 0x69, 0x22, 0x02, 0x7c, 0x2d, 0xde, 0x53, 0x0f,
0x21, 0x74, 0x4c, 0x6c, 0x1b, 0x11, 0xfb, 0x83, 0x56, 0x7f, 0x13, 0x3d, 0xac, 0xd6, 0xd4, 0x6f, 0xcb, 0x15, 0x3e, 0x84, 0x8e, 0x89, 0x6d, 0x23, 0x62, 0xbf, 0xdf, 0xec, 0x6d, 0xa2, 0x87, 0xe5,
0x80, 0x89, 0x78, 0x47, 0xd1, 0x93, 0xdd, 0x3a, 0x69, 0x45, 0x37, 0x50, 0xf1, 0x64, 0x27, 0x27, 0x8a, 0xf6, 0x2d, 0x30, 0x11, 0xef, 0x28, 0x7a, 0xb2, 0xdb, 0xc7, 0xcd, 0xe8, 0x06, 0x2a, 0x9e,
0x2d, 0xac, 0x22, 0xe6, 0x84, 0x37, 0xc1, 0xf8, 0x23, 0xec, 0x1e, 0x1c, 0x12, 0xd6, 0x2b, 0xa4, 0xec, 0xe4, 0xb8, 0x89, 0x35, 0xc4, 0x9c, 0xf0, 0x26, 0x18, 0x7f, 0x84, 0xeb, 0xfb, 0x07, 0x84,
0xc2, 0x7c, 0xb7, 0xa3, 0xcc, 0x70, 0xd8, 0x87, 0xcc, 0xae, 0xa2, 0x08, 0xa0, 0x7e, 0x6b, 0x8e, 0xf5, 0x0a, 0x29, 0x3b, 0xdf, 0x69, 0xab, 0x33, 0x1c, 0xf6, 0x01, 0xb3, 0x6b, 0x28, 0x02, 0x68,
0x8f, 0xc4, 0x54, 0xb8, 0x7f, 0xb5, 0x15, 0x85, 0x3d, 0xe7, 0x88, 0x0a, 0xb3, 0x5b, 0xae, 0xd0, 0xdf, 0x9d, 0xe3, 0x23, 0x31, 0x15, 0xee, 0x5d, 0x6d, 0x45, 0x61, 0xcf, 0x3d, 0xa4, 0xc2, 0xec,
0xb4, 0x46, 0x5e, 0xa0, 0x69, 0xad, 0x82, 0xf1, 0x47, 0xba, 0x49, 0xd1, 0xb9, 0x74, 0xcf, 0xfa, 0x96, 0x2b, 0x34, 0xad, 0x91, 0x53, 0x34, 0xad, 0x55, 0x30, 0xfe, 0xc8, 0xb0, 0x28, 0x3a, 0x93,
0xd0, 0x69, 0x72, 0x70, 0x84, 0x80, 0x15, 0xb0, 0xb0, 0x89, 0x9d, 0x80, 0xec, 0x61, 0x87, 0x94, 0xee, 0x59, 0x1f, 0xb8, 0x0d, 0x0e, 0x8e, 0x10, 0xb0, 0x04, 0x16, 0x36, 0xb1, 0x1b, 0x90, 0x5d,
0x3c, 0x82, 0x83, 0xa7, 0x4e, 0x33, 0x6a, 0x49, 0x39, 0xb1, 0x52, 0x87, 0x31, 0x48, 0x73, 0x23, 0xec, 0x92, 0xbc, 0x47, 0x70, 0xf0, 0xcc, 0x6d, 0x44, 0x2d, 0x29, 0x23, 0x56, 0xea, 0x20, 0x06,
0x94, 0x8a, 0x06, 0x31, 0x61, 0x09, 0xcc, 0x1b, 0x4d, 0x5c, 0x27, 0xae, 0xef, 0x59, 0xee, 0x11, 0xe9, 0xf5, 0x08, 0xa5, 0xa1, 0x41, 0x4c, 0x98, 0x07, 0xf3, 0x66, 0x03, 0x57, 0x49, 0xdd, 0xf7,
0xf6, 0xdb, 0x64, 0x3b, 0x64, 0xad, 0x29, 0x27, 0x1e, 0x29, 0x38, 0x82, 0x68, 0x84, 0x63, 0x54, 0xec, 0xfa, 0x21, 0xf6, 0x5b, 0x64, 0x2b, 0x64, 0xad, 0x29, 0x23, 0x1e, 0x29, 0x38, 0x82, 0xe8,
0x94, 0x65, 0xd1, 0x53, 0xc5, 0x74, 0x43, 0x82, 0x3d, 0xe1, 0x72, 0xbf, 0x98, 0x3e, 0xe6, 0x9a, 0x84, 0x63, 0x34, 0xd4, 0xcf, 0xa2, 0xa7, 0x8a, 0x55, 0x0f, 0x09, 0xf6, 0x84, 0xcb, 0xfd, 0x62,
0x0c, 0x11, 0xdf, 0x43, 0xda, 0x41, 0x33, 0x54, 0x51, 0x86, 0x06, 0x11, 0x58, 0xd0, 0x1b, 0x4f, 0xfa, 0x98, 0x6b, 0x30, 0x44, 0x7c, 0x0f, 0x69, 0x05, 0x8d, 0x50, 0x43, 0x7d, 0x34, 0x88, 0xc0,
0x71, 0x40, 0xdc, 0x10, 0x0b, 0x6a, 0xe7, 0x99, 0x9a, 0xf0, 0x71, 0x3a, 0x31, 0x28, 0x29, 0x38, 0x82, 0x51, 0x7b, 0x86, 0x03, 0x52, 0x0f, 0xb1, 0xa0, 0x76, 0x9e, 0xa9, 0x09, 0x1f, 0xa7, 0x1b,
0x88, 0x0c, 0xdf, 0x8a, 0xe7, 0x71, 0xbd, 0x4d, 0x7c, 0xcb, 0xac, 0x45, 0x2d, 0x46, 0xa8, 0x8d, 0x83, 0x92, 0x82, 0x83, 0xc8, 0xf0, 0xad, 0x78, 0x1e, 0x37, 0x5a, 0xc4, 0xb7, 0xad, 0x4a, 0xd4,
0xd3, 0x26, 0xbe, 0x46, 0xa8, 0x40, 0x12, 0x49, 0x0f, 0xdd, 0xfe, 0xfd, 0x40, 0x6f, 0x93, 0x43, 0x62, 0x84, 0xda, 0xb8, 0x2d, 0xe2, 0xeb, 0x84, 0x0a, 0x24, 0x91, 0xf4, 0xd0, 0xed, 0xdd, 0x0f,
0x59, 0x66, 0xdc, 0x21, 0x57, 0x0a, 0xa7, 0x9d, 0xba, 0x52, 0x50, 0x0a, 0xfc, 0x9a, 0x28, 0xb2, 0x8c, 0x16, 0x39, 0x50, 0x14, 0xc6, 0x1d, 0x72, 0xa5, 0x70, 0x5b, 0xa9, 0x2b, 0x05, 0xa5, 0xc0,
0xee, 0x36, 0xb1, 0x7c, 0x31, 0x7d, 0x3b, 0x66, 0xec, 0x7d, 0x97, 0x76, 0x9a, 0x14, 0xb6, 0x1f, 0x6f, 0x88, 0x22, 0xeb, 0xf5, 0x06, 0x56, 0x2e, 0xa6, 0x6f, 0xc7, 0x8c, 0xbd, 0x57, 0xa7, 0x9d,
0xfd, 0x16, 0x3e, 0x61, 0xe4, 0x4b, 0xe9, 0x9d, 0x45, 0xbf, 0x4a, 0xce, 0x4d, 0x22, 0xa1, 0x99, 0x26, 0x85, 0xed, 0x45, 0x5f, 0xc0, 0xc7, 0x8c, 0x7c, 0x29, 0xbd, 0xb3, 0xe8, 0x57, 0xc9, 0xb9,
0x99, 0xf7, 0x99, 0xc0, 0xe5, 0xf4, 0x6d, 0x44, 0x98, 0x25, 0xb9, 0xce, 0x20, 0x1a, 0xcd, 0x05, 0x49, 0x24, 0xb4, 0xfa, 0xe6, 0x7d, 0x26, 0x70, 0x39, 0x7d, 0x1b, 0x11, 0x66, 0x49, 0xae, 0x33,
0x2f, 0x17, 0x1d, 0x34, 0x59, 0x55, 0x14, 0x56, 0x15, 0x21, 0x17, 0x51, 0x8d, 0xd9, 0x80, 0xca, 0x88, 0x46, 0x73, 0xc1, 0xcb, 0x45, 0x07, 0x4d, 0x56, 0x15, 0x95, 0x55, 0x45, 0xc8, 0x45, 0x54,
0x0b, 0x92, 0xa2, 0x40, 0x0b, 0xcc, 0xf7, 0x4a, 0xd4, 0xd3, 0x59, 0x66, 0x3a, 0xc2, 0x49, 0xe6, 0x63, 0x36, 0xa0, 0xf2, 0x82, 0xa4, 0x28, 0xd0, 0x06, 0xf3, 0xdd, 0x12, 0x75, 0x75, 0x96, 0x99,
0x7a, 0x2e, 0x71, 0x9d, 0xa6, 0xd6, 0xaf, 0xb2, 0x20, 0x99, 0x15, 0xa0, 0x73, 0x00, 0xfd, 0x3b, 0x8e, 0x70, 0x92, 0xd5, 0xbd, 0x3a, 0xa9, 0xbb, 0x0d, 0xbd, 0x57, 0x65, 0x41, 0xb2, 0x5f, 0x80,
0xae, 0xef, 0x35, 0x56, 0xa3, 0xf4, 0x10, 0xdf, 0x2f, 0xb2, 0x08, 0xa6, 0xb7, 0x68, 0x76, 0x9d, 0xce, 0x01, 0xf4, 0xef, 0xb8, 0xbe, 0xd7, 0x58, 0x8d, 0xd2, 0x43, 0x7c, 0xaf, 0xc8, 0x22, 0x98,
0x48, 0x96, 0x59, 0x65, 0x12, 0xc2, 0x86, 0xe3, 0x77, 0x90, 0x4c, 0xad, 0x07, 0x70, 0xe9, 0xd8, 0xde, 0xa2, 0xd9, 0x75, 0x22, 0x59, 0x66, 0x8d, 0x49, 0x08, 0x1b, 0x8e, 0xdf, 0x41, 0xfa, 0x6a,
0x1d, 0x5f, 0x50, 0x58, 0xbe, 0xaf, 0x0f, 0xbf, 0xcf, 0xf0, 0x74, 0x27, 0xe0, 0xf1, 0xcb, 0xc4, 0x3d, 0x80, 0x4b, 0xc7, 0xee, 0xf8, 0x82, 0xc2, 0xf2, 0x7d, 0x7d, 0xf8, 0x7d, 0x86, 0xa7, 0x3b,
0xe5, 0x7e, 0x69, 0xe8, 0x8d, 0x84, 0x93, 0x45, 0x30, 0xdc, 0x4e, 0xdd, 0x20, 0x98, 0xc2, 0x8d, 0x01, 0x8f, 0x5f, 0x26, 0x2e, 0xf7, 0x4b, 0x43, 0x6f, 0x24, 0x9c, 0x2c, 0x82, 0xe1, 0x56, 0xea,
0xe7, 0x5d, 0x20, 0xb8, 0x50, 0x96, 0x49, 0xc7, 0xbb, 0x12, 0x2f, 0x45, 0xb1, 0xd9, 0x66, 0xff, 0x06, 0xc1, 0x14, 0x6e, 0xbc, 0xe8, 0x02, 0xc1, 0x85, 0xfa, 0x99, 0x74, 0xbc, 0xcb, 0xf3, 0x52,
0x2b, 0x78, 0x33, 0xbd, 0x77, 0xe2, 0x52, 0xd5, 0x39, 0x40, 0x45, 0x29, 0x06, 0xfd, 0xa2, 0x93, 0xe4, 0x1a, 0x2d, 0xf6, 0xbf, 0x82, 0x37, 0xd3, 0x7b, 0x27, 0x2e, 0x55, 0x95, 0x03, 0x34, 0x94,
0x96, 0x1a, 0x71, 0x08, 0x8e, 0xa6, 0x0e, 0x21, 0xc1, 0x29, 0x21, 0x2d, 0xa4, 0x30, 0x15, 0x0d, 0x62, 0xd0, 0x2f, 0x3a, 0x69, 0xa9, 0x10, 0x97, 0xe0, 0x68, 0xea, 0x10, 0x12, 0x9c, 0x12, 0xd2,
0x22, 0x67, 0x35, 0x2d, 0xff, 0x09, 0xf6, 0xe4, 0x57, 0x9e, 0xa7, 0x49, 0x28, 0x2c, 0xa3, 0xc9, 0x43, 0x0a, 0xd3, 0xd0, 0x20, 0x72, 0xbf, 0xa6, 0xed, 0x3f, 0xc5, 0x9e, 0xf2, 0xca, 0x8b, 0x34,
0xc8, 0xf0, 0x01, 0x98, 0x89, 0xef, 0x30, 0x45, 0xbf, 0xed, 0x11, 0xf9, 0x1e, 0x3b, 0x0b, 0xc5, 0x09, 0x85, 0xf5, 0x69, 0x32, 0x32, 0x7c, 0x00, 0x66, 0xe2, 0x3b, 0x4c, 0xce, 0x6f, 0x79, 0x44,
0xe6, 0x15, 0x5f, 0x96, 0xea, 0xd4, 0x4f, 0x9b, 0x97, 0x88, 0x87, 0x26, 0x98, 0x7f, 0xd8, 0xf6, 0xb9, 0xc7, 0xce, 0x42, 0xb1, 0x79, 0xc5, 0x97, 0xa5, 0x2a, 0xf5, 0xd3, 0xe6, 0x25, 0xe2, 0xa1,
0x89, 0x53, 0x70, 0xea, 0x4f, 0xb0, 0xd7, 0x28, 0x9c, 0x10, 0x1c, 0xca, 0xaf, 0x31, 0x11, 0x61, 0x05, 0xe6, 0x1f, 0xb6, 0x7c, 0xe2, 0x66, 0xdd, 0xea, 0x53, 0xec, 0xd5, 0xb2, 0xc7, 0x04, 0x87,
0xd6, 0xff, 0x80, 0x42, 0xb4, 0x3d, 0x8e, 0xd1, 0xf6, 0x28, 0x48, 0x45, 0x59, 0x22, 0x6d, 0x25, 0xca, 0x6b, 0x4c, 0x44, 0x98, 0xf5, 0xdf, 0xa7, 0x10, 0x7d, 0x97, 0x63, 0xf4, 0x5d, 0x0a, 0xd2,
0xd5, 0x00, 0xef, 0xfa, 0x04, 0xcb, 0x0f, 0xd2, 0xc7, 0x55, 0x2b, 0xc0, 0xda, 0x53, 0x9f, 0x66, 0x50, 0x3f, 0x91, 0xb6, 0x92, 0x72, 0x80, 0x77, 0x7c, 0x82, 0x95, 0x07, 0xe9, 0xe3, 0xaa, 0x19,
0x27, 0xc6, 0x88, 0x19, 0xf1, 0x83, 0xa0, 0xdd, 0x22, 0x6c, 0x62, 0x92, 0xdf, 0x4b, 0x6f, 0xe3, 0x60, 0xfd, 0x99, 0x4f, 0xb3, 0x13, 0x63, 0xc4, 0x8c, 0xf8, 0x41, 0xd0, 0x6a, 0x12, 0x36, 0x31,
0x5e, 0x46, 0x38, 0x4a, 0x63, 0x33, 0x96, 0x90, 0x11, 0x81, 0x4c, 0xdb, 0xa4, 0xe9, 0x1f, 0x1c, 0x29, 0xef, 0xa5, 0xb7, 0x71, 0x37, 0x23, 0x1c, 0xa5, 0xb3, 0x19, 0x4b, 0xc8, 0x88, 0x40, 0xa6,
0xe0, 0x40, 0xde, 0x60, 0x89, 0x15, 0xda, 0x64, 0x93, 0xd9, 0x55, 0x14, 0x01, 0xe8, 0xfd, 0xc1, 0x6d, 0xd2, 0xf2, 0xf7, 0xf7, 0x71, 0xa0, 0x6c, 0xb0, 0xc4, 0x0a, 0x6d, 0xb2, 0xc1, 0xec, 0x1a,
0xf4, 0x0f, 0x2a, 0x6d, 0xd2, 0x6a, 0x93, 0x50, 0xde, 0x64, 0xdf, 0xb3, 0x70, 0x7f, 0x68, 0xfa, 0x8a, 0x00, 0xf4, 0xfe, 0x60, 0xf9, 0xfb, 0xa5, 0x16, 0x69, 0xb6, 0x48, 0xa8, 0x6c, 0xb2, 0xef,
0x07, 0x9a, 0xcf, 0x9d, 0x2a, 0x12, 0x90, 0xf0, 0x0e, 0x98, 0x30, 0xfd, 0x03, 0x13, 0x3f, 0xc5, 0x59, 0xb8, 0x3f, 0x34, 0xfc, 0x7d, 0xdd, 0xe7, 0x4e, 0x0d, 0x09, 0x48, 0x78, 0x07, 0x4c, 0x58,
0x4d, 0xb9, 0x94, 0x3e, 0x14, 0x29, 0xab, 0x49, 0x5d, 0x2a, 0xea, 0xa1, 0x56, 0xff, 0x2b, 0x81, 0xfe, 0xbe, 0x85, 0x9f, 0xe1, 0x86, 0x92, 0x4f, 0x1f, 0x8a, 0x94, 0xd5, 0xa0, 0x2e, 0x0d, 0x75,
0xe9, 0xb8, 0xdb, 0xb3, 0x66, 0x0e, 0xc1, 0xec, 0xd6, 0xae, 0xfd, 0x08, 0x95, 0x2c, 0xc3, 0xae, 0x51, 0xab, 0xff, 0x95, 0xc0, 0x74, 0xdc, 0xed, 0x59, 0x33, 0x87, 0x60, 0xb6, 0xb0, 0xe3, 0x3c,
0x6d, 0xeb, 0xa6, 0x99, 0x3f, 0x93, 0xb0, 0x99, 0x3a, 0xda, 0x30, 0xf2, 0x12, 0x5c, 0x00, 0x73, 0x42, 0x79, 0xdb, 0x74, 0x2a, 0x5b, 0x86, 0x65, 0xc9, 0x67, 0x12, 0x36, 0xcb, 0x40, 0x1b, 0xa6,
0x5b, 0xbb, 0x36, 0x32, 0xf4, 0x35, 0xbb, 0x52, 0x36, 0xec, 0x2d, 0xe3, 0xfd, 0xfc, 0x08, 0x9c, 0x2c, 0xc1, 0x05, 0x30, 0x57, 0xd8, 0x71, 0x90, 0x69, 0xac, 0x39, 0xa5, 0xa2, 0xe9, 0x14, 0xcc,
0x07, 0x33, 0xb1, 0x11, 0xe9, 0xe5, 0x0d, 0x23, 0x9f, 0x83, 0x8b, 0x60, 0x7e, 0x6b, 0xd7, 0x5e, 0x27, 0xf2, 0x08, 0x9c, 0x07, 0x33, 0xb1, 0x11, 0x19, 0xc5, 0x0d, 0x53, 0xce, 0xc0, 0x45, 0x30,
0x33, 0x4c, 0xc3, 0x32, 0x7a, 0xc8, 0xd1, 0x88, 0x1e, 0x99, 0x39, 0x76, 0x0c, 0x5e, 0x00, 0x0b, 0x5f, 0xd8, 0x71, 0xd6, 0x4c, 0xcb, 0xb4, 0xcd, 0x2e, 0x72, 0x34, 0xa2, 0x47, 0x66, 0x8e, 0x1d,
0x5b, 0xbb, 0xb6, 0xf5, 0xb8, 0x1c, 0xad, 0xc5, 0xdd, 0xf9, 0x71, 0x38, 0x09, 0xc6, 0x4c, 0x43, 0x83, 0x17, 0xc0, 0x42, 0x61, 0xc7, 0xb1, 0x1f, 0x17, 0xa3, 0xb5, 0xb8, 0x5b, 0x1e, 0x87, 0x93,
0xaf, 0x19, 0x79, 0x40, 0x89, 0x86, 0x69, 0x14, 0xad, 0x52, 0xa5, 0x6c, 0xa3, 0x9d, 0x72, 0xd9, 0x60, 0xcc, 0x32, 0x8d, 0x8a, 0x29, 0x03, 0x4a, 0x34, 0x2d, 0x33, 0x67, 0xe7, 0x4b, 0x45, 0x07,
0x40, 0xf9, 0x73, 0x30, 0x0f, 0xa6, 0x1f, 0xe9, 0x56, 0x71, 0x33, 0xb6, 0x28, 0x74, 0x59, 0xb3, 0x6d, 0x17, 0x8b, 0x26, 0x92, 0xcf, 0x41, 0x19, 0x4c, 0x3f, 0x32, 0xec, 0xdc, 0x66, 0x6c, 0x51,
0x52, 0xdc, 0xb2, 0x91, 0x5e, 0x34, 0x50, 0x6c, 0xbe, 0x49, 0x81, 0x4c, 0x28, 0xb6, 0xdc, 0x5b, 0xe9, 0xb2, 0x56, 0x29, 0x57, 0x70, 0x90, 0x91, 0x33, 0x51, 0x6c, 0xbe, 0x49, 0x81, 0x4c, 0x28,
0x2d, 0x80, 0xb3, 0xd1, 0x34, 0x0c, 0xa7, 0xc0, 0xd9, 0xad, 0x5d, 0x7b, 0x53, 0xaf, 0x6d, 0xe6, 0xb6, 0xdc, 0x5b, 0xfd, 0x36, 0x38, 0x1b, 0x4d, 0xc3, 0x70, 0x0a, 0x9c, 0x2d, 0xec, 0x38, 0x9b,
0xcf, 0xf4, 0x91, 0xc6, 0xe3, 0x6a, 0x09, 0xd1, 0x37, 0x06, 0x60, 0x3c, 0x62, 0x8d, 0xc0, 0x69, 0x46, 0x65, 0x53, 0x3e, 0xd3, 0x43, 0x9a, 0x8f, 0xcb, 0x79, 0x44, 0xdf, 0x18, 0x80, 0xf1, 0x88,
0x30, 0x51, 0xae, 0xd8, 0xc5, 0x4d, 0xa3, 0xb8, 0x95, 0xcf, 0xad, 0xfe, 0x38, 0x27, 0xfc, 0x7a, 0x35, 0x02, 0xa7, 0xc1, 0x44, 0xb1, 0xe4, 0xe4, 0x36, 0xcd, 0x5c, 0x41, 0xce, 0xc0, 0x4b, 0xe0,
0x00, 0xe7, 0xc0, 0x54, 0xb9, 0x62, 0xd9, 0x35, 0x4b, 0x47, 0x96, 0xb1, 0x96, 0x3f, 0x03, 0xcf, 0x7c, 0x65, 0xb3, 0x84, 0x6c, 0xc7, 0xb6, 0x2d, 0x27, 0xc1, 0x1a, 0x5d, 0xfd, 0x49, 0x46, 0xf8,
0x03, 0x58, 0x2a, 0x97, 0xac, 0x92, 0x6e, 0x72, 0xa3, 0x6d, 0x58, 0xc5, 0xb5, 0x3c, 0xa0, 0x4b, 0x65, 0x01, 0xce, 0x81, 0xa9, 0x62, 0xc9, 0x76, 0x2a, 0xb6, 0x81, 0x6c, 0x73, 0x4d, 0x3e, 0x03,
0x20, 0x43, 0xb0, 0x4c, 0x51, 0x4b, 0xad, 0xb4, 0x61, 0x19, 0x68, 0x9b, 0x5b, 0xce, 0xc1, 0x65, 0xcf, 0x03, 0x98, 0x2f, 0xe6, 0xed, 0xbc, 0x61, 0x71, 0xa3, 0x63, 0xda, 0xb9, 0x35, 0x19, 0xd0,
0x70, 0xa5, 0x56, 0xda, 0x78, 0xb8, 0x53, 0xe2, 0x18, 0x5b, 0x2f, 0xaf, 0xd9, 0xc8, 0xd8, 0xae, 0xe5, 0x91, 0x29, 0x58, 0xa6, 0xa8, 0xa5, 0x92, 0xdf, 0xb0, 0x4d, 0xb4, 0xc5, 0x2d, 0xe7, 0xe0,
0xec, 0x1a, 0xf6, 0x9a, 0x6e, 0xe9, 0xf9, 0x45, 0x9a, 0xf3, 0x9a, 0xbe, 0x6b, 0xd8, 0xb5, 0xb2, 0x32, 0xb8, 0x52, 0xc9, 0x6f, 0x3c, 0xdc, 0xce, 0x73, 0x8c, 0x63, 0x14, 0xd7, 0x1c, 0x64, 0x6e,
0x5e, 0xad, 0x6d, 0x56, 0xac, 0xfc, 0x12, 0xbc, 0x06, 0xae, 0x52, 0xe1, 0x0a, 0x32, 0xec, 0x78, 0x95, 0x76, 0x4c, 0x67, 0xcd, 0xb0, 0x0d, 0x79, 0x91, 0xd6, 0xa3, 0x62, 0xec, 0x98, 0x4e, 0xa5,
0x81, 0x75, 0x54, 0xd9, 0xee, 0x43, 0x14, 0x78, 0x11, 0x2c, 0x0e, 0x76, 0x2d, 0x53, 0x76, 0x66, 0x68, 0x94, 0x2b, 0x9b, 0x25, 0x5b, 0x5e, 0x82, 0xd7, 0xc0, 0x55, 0x2a, 0x5c, 0x42, 0xa6, 0x13,
0x49, 0x1d, 0x15, 0x37, 0x4b, 0xf1, 0x9a, 0x2b, 0xf0, 0x36, 0x78, 0xe5, 0xb4, 0xa8, 0xd8, 0x73, 0x2f, 0xb0, 0x8e, 0x4a, 0x5b, 0x3d, 0x88, 0x0a, 0x2f, 0x82, 0xc5, 0xc1, 0xae, 0x65, 0xca, 0xee,
0xcd, 0xaa, 0x54, 0x6d, 0x7d, 0xc3, 0x28, 0x5b, 0xf9, 0x9b, 0xf0, 0x2a, 0xb8, 0x58, 0x30, 0xf5, 0x5b, 0xd2, 0x40, 0xb9, 0xcd, 0x7c, 0xbc, 0xe6, 0x0a, 0xbc, 0x0d, 0x5e, 0x39, 0x29, 0x2a, 0xf6,
0xe2, 0xd6, 0x66, 0xc5, 0x34, 0xec, 0xaa, 0x61, 0x20, 0xbb, 0x5a, 0x41, 0x96, 0x6d, 0x3d, 0xb6, 0x5c, 0xb1, 0x4b, 0x65, 0xc7, 0xd8, 0x30, 0x8b, 0xb6, 0x7c, 0x13, 0x5e, 0x05, 0x17, 0xb3, 0x96,
0xd1, 0xe3, 0x7c, 0x03, 0x2a, 0xe0, 0xf2, 0x4e, 0x79, 0x38, 0x00, 0xc3, 0x4b, 0x60, 0x71, 0xcd, 0x91, 0x2b, 0x6c, 0x96, 0x2c, 0xd3, 0x29, 0x9b, 0x26, 0x72, 0xca, 0x2c, 0x97, 0x8f, 0x1d, 0xf4,
0x30, 0xf5, 0xf7, 0x33, 0xae, 0x67, 0x12, 0xbc, 0x02, 0x2e, 0xec, 0x94, 0x07, 0x7b, 0x3f, 0x91, 0x58, 0xae, 0x41, 0x15, 0x5c, 0xde, 0x2e, 0x0e, 0x07, 0x60, 0x78, 0x09, 0x2c, 0xae, 0x99, 0x96,
0x56, 0xff, 0x0a, 0xc0, 0x28, 0xbd, 0x3e, 0x42, 0x19, 0x9c, 0x8b, 0x73, 0x4b, 0xb7, 0xe1, 0x7a, 0xf1, 0xa4, 0xcf, 0xf5, 0x5c, 0x82, 0x57, 0xc0, 0x85, 0xed, 0xe2, 0x60, 0xef, 0xa7, 0xd2, 0xea,
0xc5, 0x34, 0x2b, 0x8f, 0x0c, 0x94, 0x3f, 0x13, 0xbd, 0x4d, 0xc6, 0x63, 0xef, 0x94, 0xad, 0x92, 0xc7, 0x53, 0x60, 0x94, 0x5e, 0x2d, 0xa1, 0x02, 0xce, 0xc5, 0xb9, 0xa5, 0x5b, 0x74, 0xbd, 0x64,
0x69, 0x5b, 0xa8, 0xb4, 0xb1, 0x61, 0xa0, 0x7e, 0x86, 0x24, 0xfa, 0x3d, 0xc4, 0x04, 0xd3, 0xd0, 0x59, 0xa5, 0x47, 0x26, 0x92, 0xcf, 0x44, 0x6f, 0xd3, 0xe7, 0x71, 0xb6, 0x8b, 0x76, 0xde, 0x72,
0xd7, 0xd8, 0x8e, 0xb8, 0x09, 0x6e, 0x24, 0x6d, 0xc3, 0xe8, 0x39, 0x91, 0xfe, 0x70, 0xa7, 0x82, 0x6c, 0x94, 0xdf, 0xd8, 0x30, 0x51, 0x2f, 0x43, 0x12, 0xfd, 0x56, 0x62, 0x82, 0x65, 0x1a, 0x6b,
0x76, 0xb6, 0xf3, 0xa3, 0x74, 0xd3, 0xc4, 0x36, 0xfa, 0xcd, 0x8d, 0xc1, 0xeb, 0x40, 0x89, 0x53, 0x6c, 0xb7, 0xdc, 0x04, 0x37, 0x92, 0xb6, 0x61, 0xf4, 0x8c, 0x48, 0x7f, 0xb8, 0x5d, 0x42, 0xdb,
0x2c, 0x64, 0x37, 0x11, 0x39, 0x80, 0xf7, 0xc1, 0xeb, 0xcf, 0x01, 0x0d, 0x8b, 0x62, 0x8a, 0x96, 0x5b, 0xf2, 0x28, 0xdd, 0x34, 0xb1, 0x8d, 0x7e, 0x8f, 0x63, 0xf0, 0x3a, 0x50, 0xe3, 0x14, 0x0b,
0x64, 0x00, 0x37, 0x7a, 0x9f, 0x69, 0xf8, 0x1a, 0xb8, 0x33, 0xd4, 0x3d, 0x4c, 0x74, 0x06, 0xae, 0xd9, 0x4d, 0x44, 0x0e, 0xe0, 0x7d, 0xf0, 0xfa, 0x0b, 0x40, 0xc3, 0xa2, 0x98, 0xa2, 0x25, 0x19,
0x83, 0xc2, 0x00, 0x16, 0x7f, 0xcb, 0xc8, 0xc2, 0xf7, 0x65, 0x24, 0x14, 0x53, 0xa3, 0x4d, 0x58, 0xc0, 0x8d, 0xde, 0x67, 0x1a, 0xbe, 0x06, 0xee, 0x0c, 0x75, 0x0f, 0x13, 0x9d, 0x81, 0xeb, 0x20,
0x44, 0xf4, 0x2b, 0xce, 0xcf, 0xc2, 0x55, 0xf0, 0xf2, 0xd0, 0xed, 0x90, 0x4c, 0x42, 0x03, 0xea, 0x3b, 0x80, 0xc5, 0xdf, 0x32, 0xb2, 0xf0, 0x7d, 0x19, 0x09, 0xc5, 0xd4, 0x68, 0x13, 0xe6, 0x10,
0xe0, 0x9d, 0x17, 0xc3, 0x0e, 0x0b, 0x1b, 0xc3, 0x97, 0xc0, 0xf2, 0x70, 0x89, 0x28, 0x25, 0xfb, 0xfd, 0xc2, 0xe5, 0x59, 0xb8, 0x0a, 0x5e, 0x1e, 0xba, 0x1d, 0x92, 0x49, 0xa8, 0x41, 0x03, 0xbc,
0xf0, 0x6d, 0xf0, 0xc6, 0xf3, 0x50, 0xc3, 0x96, 0x38, 0x38, 0x7d, 0x89, 0x68, 0x1b, 0x1c, 0xd2, 0x73, 0x3a, 0xec, 0xb0, 0xb0, 0x31, 0x7c, 0x09, 0x2c, 0x0f, 0x97, 0x88, 0x52, 0xb2, 0x07, 0xdf,
0x6f, 0x6f, 0x38, 0x8a, 0x6e, 0x0c, 0x17, 0xfe, 0x1f, 0x50, 0x07, 0x6e, 0xf6, 0x64, 0x5a, 0x9e, 0x06, 0x6f, 0xbc, 0x08, 0x35, 0x6c, 0x89, 0xfd, 0x93, 0x97, 0x88, 0xb6, 0xc1, 0x01, 0xfd, 0xf6,
0x49, 0xf0, 0x16, 0xb8, 0x89, 0xf4, 0xf2, 0x5a, 0x65, 0xdb, 0x7e, 0x01, 0xfc, 0x27, 0x12, 0x7c, 0x86, 0xa3, 0xe8, 0xc6, 0xa8, 0xc3, 0xff, 0x03, 0xda, 0xc0, 0xcd, 0x9e, 0x4c, 0xcb, 0x73, 0x09,
0x17, 0xbc, 0xf5, 0x7c, 0xe0, 0xb0, 0x17, 0xfc, 0x54, 0x82, 0x06, 0x78, 0xef, 0x85, 0xd7, 0x1b, 0xde, 0x02, 0x37, 0x91, 0x51, 0x5c, 0x2b, 0x6d, 0x39, 0xa7, 0xc0, 0x7f, 0x2a, 0xc1, 0x77, 0xc1,
0x26, 0xf3, 0x99, 0x04, 0xaf, 0x81, 0x2b, 0x83, 0xf9, 0x51, 0x1d, 0x3e, 0x97, 0xe0, 0x0a, 0xb8, 0x5b, 0x2f, 0x06, 0x0e, 0x7b, 0xc1, 0xcf, 0x24, 0x68, 0x82, 0xf7, 0x4e, 0xbd, 0xde, 0x30, 0x99,
0x7e, 0xea, 0x4a, 0x11, 0xf2, 0x0b, 0x09, 0xbe, 0x09, 0xee, 0x9d, 0x06, 0x19, 0x16, 0xc6, 0xaf, 0xcf, 0x25, 0x78, 0x0d, 0x5c, 0x19, 0xcc, 0x8f, 0xea, 0xf0, 0x85, 0x04, 0x57, 0xc0, 0xf5, 0x13,
0x24, 0xf8, 0x00, 0xdc, 0x7f, 0x81, 0x35, 0x86, 0x09, 0xfc, 0xfa, 0x94, 0xf7, 0x88, 0x8a, 0xfd, 0x57, 0x8a, 0x90, 0x5f, 0x4a, 0xf0, 0x4d, 0x70, 0xef, 0x24, 0xc8, 0xb0, 0x30, 0x7e, 0x2d, 0xc1,
0xe5, 0xf3, 0xdf, 0x23, 0x42, 0xfe, 0x46, 0x82, 0x4b, 0xe0, 0xe2, 0x60, 0x08, 0xdd, 0x13, 0xbf, 0x07, 0xe0, 0xfe, 0x29, 0xd6, 0x18, 0x26, 0xf0, 0x9b, 0x13, 0xde, 0x23, 0x2a, 0xf6, 0x57, 0x2f,
0x95, 0xe0, 0x0d, 0xb0, 0x7c, 0xaa, 0x12, 0x85, 0xfd, 0x4e, 0x82, 0x32, 0x58, 0x28, 0x57, 0xec, 0x7e, 0x8f, 0x08, 0xf9, 0x5b, 0x09, 0x2e, 0x81, 0x8b, 0x83, 0x21, 0x74, 0x4f, 0xfc, 0x4e, 0x82,
0x75, 0xbd, 0x64, 0xda, 0x8f, 0x4a, 0xd6, 0xa6, 0x5d, 0xb3, 0x90, 0x51, 0xab, 0xe5, 0x7f, 0x3a, 0x37, 0xc0, 0xf2, 0x89, 0x4a, 0x14, 0xf6, 0x7b, 0x09, 0x2a, 0x60, 0xa1, 0x58, 0x72, 0xd6, 0x8d,
0x42, 0x43, 0x49, 0x78, 0xca, 0x95, 0xc8, 0x69, 0xaf, 0x57, 0x90, 0x6d, 0x96, 0x76, 0x8d, 0x32, 0xbc, 0xe5, 0x3c, 0xca, 0xdb, 0x9b, 0x4e, 0xc5, 0x46, 0x66, 0xa5, 0x22, 0xff, 0x6c, 0x84, 0x86,
0x45, 0x7e, 0x3c, 0x02, 0xe7, 0x00, 0xa0, 0xb0, 0x6a, 0xa5, 0x54, 0xb6, 0x6a, 0xf9, 0x6f, 0xe7, 0x92, 0xf0, 0x14, 0x4b, 0x91, 0xd3, 0x59, 0x2f, 0x21, 0xc7, 0xca, 0xef, 0x98, 0x45, 0x8a, 0xfc,
0xe0, 0x0c, 0x98, 0x30, 0x1e, 0x5b, 0x06, 0x2a, 0xeb, 0x66, 0xfe, 0x6f, 0xb9, 0xbb, 0x0f, 0xc0, 0x64, 0x04, 0xce, 0x01, 0x40, 0x61, 0xe5, 0x52, 0xbe, 0x68, 0x57, 0xe4, 0xef, 0x65, 0xe0, 0x4b,
0xa4, 0x15, 0x38, 0x5e, 0xd8, 0xf2, 0x03, 0x02, 0xef, 0x8a, 0x0f, 0xb3, 0xd1, 0xff, 0x67, 0x45, 0x40, 0xed, 0x19, 0x38, 0x7b, 0x2d, 0x5f, 0x29, 0x38, 0xf9, 0x92, 0x63, 0x19, 0xb6, 0x59, 0xcc,
0xbf, 0xb9, 0x5f, 0x9a, 0xeb, 0x3d, 0xf3, 0x9f, 0x63, 0xd5, 0x33, 0x2b, 0xd2, 0x1d, 0xa9, 0x70, 0x3d, 0x91, 0x3f, 0xca, 0xc0, 0x19, 0x30, 0x61, 0x3e, 0xb6, 0x4d, 0x54, 0x34, 0x2c, 0xf9, 0x6f,
0xee, 0xd9, 0x9f, 0x96, 0xce, 0x3c, 0xfb, 0x6a, 0x49, 0xfa, 0xf2, 0xab, 0x25, 0xe9, 0x8f, 0x5f, 0x99, 0xbb, 0x0f, 0xc0, 0xa4, 0x1d, 0xb8, 0x5e, 0xd8, 0xf4, 0x03, 0x02, 0xef, 0x8a, 0x0f, 0xb3,
0x2d, 0x49, 0x3f, 0xfa, 0xf3, 0xd2, 0x99, 0xbd, 0x71, 0xf6, 0x9b, 0xfd, 0xbd, 0xff, 0x05, 0x00, 0xd1, 0xff, 0x88, 0x45, 0xbf, 0xda, 0x5f, 0x9a, 0xeb, 0x3e, 0xf3, 0x1f, 0x74, 0xb5, 0x33, 0x2b,
0x00, 0xff, 0xff, 0x44, 0x6a, 0x6d, 0x90, 0xfc, 0x1f, 0x00, 0x00, 0xd2, 0x1d, 0x29, 0x7b, 0xee, 0xf9, 0x9f, 0x97, 0xce, 0x3c, 0xff, 0x7a, 0x49, 0xfa, 0xea, 0xeb,
0x25, 0xe9, 0x4f, 0x5f, 0x2f, 0x49, 0x3f, 0xfe, 0xcb, 0xd2, 0x99, 0xdd, 0x71, 0xf6, 0xab, 0xff,
0xbd, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x21, 0xcd, 0x4e, 0x90, 0x3e, 0x20, 0x00, 0x00,
} }
// Reference imports to suppress errors if they are not otherwise used. // Reference imports to suppress errors if they are not otherwise used.
@ -2972,10 +2981,7 @@ func (m *Request) Unmarshal(dAtA []byte) error {
if err != nil { if err != nil {
return err return err
} }
if skippy < 0 { if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthRpc
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthRpc return ErrInvalidLengthRpc
} }
if (iNdEx + skippy) > l { if (iNdEx + skippy) > l {
@ -3275,10 +3281,7 @@ func (m *SnapshotInfo) Unmarshal(dAtA []byte) error {
if err != nil { if err != nil {
return err return err
} }
if skippy < 0 { if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthRpc
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthRpc return ErrInvalidLengthRpc
} }
if (iNdEx + skippy) > l { if (iNdEx + skippy) > l {
@ -3453,10 +3456,7 @@ func (m *Response) Unmarshal(dAtA []byte) error {
if err != nil { if err != nil {
return err return err
} }
if skippy < 0 { if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthRpc
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthRpc return ErrInvalidLengthRpc
} }
if (iNdEx + skippy) > l { if (iNdEx + skippy) > l {
@ -4263,10 +4263,7 @@ func (m *Member) Unmarshal(dAtA []byte) error {
if err != nil { if err != nil {
return err return err
} }
if skippy < 0 { if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthRpc
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthRpc return ErrInvalidLengthRpc
} }
if (iNdEx + skippy) > l { if (iNdEx + skippy) > l {
@ -4895,10 +4892,7 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
if err != nil { if err != nil {
return err return err
} }
if skippy < 0 { if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthRpc
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthRpc return ErrInvalidLengthRpc
} }
if (iNdEx + skippy) > l { if (iNdEx + skippy) > l {
@ -4992,10 +4986,7 @@ func (m *Stresser) Unmarshal(dAtA []byte) error {
if err != nil { if err != nil {
return err return err
} }
if skippy < 0 { if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthRpc
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthRpc return ErrInvalidLengthRpc
} }
if (iNdEx + skippy) > l { if (iNdEx + skippy) > l {
@ -5850,10 +5841,7 @@ func (m *Etcd) Unmarshal(dAtA []byte) error {
if err != nil { if err != nil {
return err return err
} }
if skippy < 0 { if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthRpc
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthRpc return ErrInvalidLengthRpc
} }
if (iNdEx + skippy) > l { if (iNdEx + skippy) > l {

4
tests/functional/rpcpb/rpc.proto
View File

@ -194,6 +194,7 @@ enum Checker {
LEASE_EXPIRE = 1; LEASE_EXPIRE = 1;
RUNNER = 2; RUNNER = 2;
NO_CHECK = 3; NO_CHECK = 3;
SHORT_TTL_LEASE_EXPIRE = 4;
} }
message Etcd { message Etcd {
@ -625,6 +626,9 @@ enum Case {
// in critical code paths. // in critical code paths.
FAILPOINTS = 400; FAILPOINTS = 400;
// FAILPOINTS_WITH_DISK_IO_LATENCY injects high disk I/O latency failure in raftAfterSave code paths.
FAILPOINTS_WITH_DISK_IO_LATENCY = 401;
// EXTERNAL runs external failure injection scripts. // EXTERNAL runs external failure injection scripts.
EXTERNAL = 500; EXTERNAL = 500;
} }

71
tests/functional/tester/case_failpoints_disk_io.go Normal file
View File

@ -0,0 +1,71 @@
// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tester
import (
"fmt"
"strings"
"go.etcd.io/etcd/tests/v3/functional/rpcpb"
)
const (
diskIOFailpoint = "raftAfterSave"
)
func failpointDiskIOFailures(clus *Cluster) (ret []Case, err error) {
fps, err := failpointPaths(clus.Members[0].FailpointHTTPAddr)
if err != nil {
return nil, err
}
var detailDiskIOLatencyFailpointPath string
for i := 0; i < len(fps); i++ {
if strings.HasSuffix(fps[i], diskIOFailpoint) {
detailDiskIOLatencyFailpointPath = fps[i]
break
}
}
// create failure objects for diskIOFailpoint
fpFails := casesFromDiskIOFailpoint(detailDiskIOLatencyFailpointPath, clus.Tester.FailpointCommands)
// wrap in delays so failpoint has time to trigger
for i, fpf := range fpFails {
fpFails[i] = &caseDelay{
Case: fpf,
delayDuration: clus.GetCaseDelayDuration(),
}
}
ret = append(ret, fpFails...)
return ret, nil
}
func casesFromDiskIOFailpoint(fp string, failpointCommands []string) (fs []Case) {
recov := makeRecoverFailpoint(fp)
for _, fcmd := range failpointCommands {
inject := makeInjectFailpoint(fp, fcmd)
fs = append(fs, []Case{
&caseLeader{
caseByFunc: caseByFunc{
desc: fmt.Sprintf("failpoint %q (leader: %q)", fp, fcmd),
rpcpbCase: rpcpb.Case_FAILPOINTS,
injectMember: inject,
recoverMember: recov,
},
last: -1,
lead: -1,
},
}...)
}
return fs
}

47
tests/functional/tester/checker_lease_expire.go
View File

@ -74,12 +74,13 @@ func (lc *leaseExpireChecker) Check() error {
}() }()
lc.cli = cli lc.cli = cli
if err := lc.check(true, lc.ls.revokedLeases.leases); err != nil { if err := check(lc.lg, lc.cli, true, lc.ls.revokedLeases.leases); err != nil {
return err return err
} }
if err := lc.check(false, lc.ls.aliveLeases.leases); err != nil { if err := check(lc.lg, lc.cli, false, lc.ls.aliveLeases.leases); err != nil {
return err return err
} }
return lc.checkShortLivedLeases() return lc.checkShortLivedLeases()
} }
@ -109,7 +110,7 @@ func (lc *leaseExpireChecker) checkShortLivedLease(ctx context.Context, leaseID
// retry in case of transient failure or lease is expired but not yet revoked due to the fact that etcd cluster didn't have enought time to delete it. // retry in case of transient failure or lease is expired but not yet revoked due to the fact that etcd cluster didn't have enought time to delete it.
var resp *clientv3.LeaseTimeToLiveResponse var resp *clientv3.LeaseTimeToLiveResponse
for i := 0; i < retries; i++ { for i := 0; i < retries; i++ {
resp, err = lc.getLeaseByID(ctx, leaseID) resp, err = getLeaseByID(ctx, lc.cli, leaseID)
// lease not found, for ~v3.1 compatibilities, check ErrLeaseNotFound // lease not found, for ~v3.1 compatibilities, check ErrLeaseNotFound
if (err == nil && resp.TTL == -1) || (err != nil && rpctypes.Error(err) == rpctypes.ErrLeaseNotFound) { if (err == nil && resp.TTL == -1) || (err != nil && rpctypes.Error(err) == rpctypes.ErrLeaseNotFound) {
return nil return nil
@ -142,7 +143,7 @@ func (lc *leaseExpireChecker) checkShortLivedLease(ctx context.Context, leaseID
) )
time.Sleep(time.Second) time.Sleep(time.Second)
} }
if err = lc.checkLease(ctx, false, leaseID); err != nil { if err = checkLease(ctx, lc.lg, lc.cli, false, leaseID); err != nil {
continue continue
} }
return nil return nil
@ -150,21 +151,21 @@ func (lc *leaseExpireChecker) checkShortLivedLease(ctx context.Context, leaseID
return err return err
} }
func (lc *leaseExpireChecker) checkLease(ctx context.Context, expired bool, leaseID int64) error { func checkLease(ctx context.Context, lg *zap.Logger, cli *clientv3.Client, expired bool, leaseID int64) error {
keysExpired, err := lc.hasKeysAttachedToLeaseExpired(ctx, leaseID) keysExpired, err := hasKeysAttachedToLeaseExpired(ctx, lg, cli, leaseID)
if err != nil { if err != nil {
lc.lg.Warn( lg.Warn(
"hasKeysAttachedToLeaseExpired failed", "hasKeysAttachedToLeaseExpired failed",
zap.String("endpoint", lc.m.EtcdClientEndpoint), zap.Any("endpoint", cli.Endpoints()),
zap.Error(err), zap.Error(err),
) )
return err return err
} }
leaseExpired, err := lc.hasLeaseExpired(ctx, leaseID) leaseExpired, err := hasLeaseExpired(ctx, lg, cli, leaseID)
if err != nil { if err != nil {
lc.lg.Warn( lg.Warn(
"hasLeaseExpired failed", "hasLeaseExpired failed",
zap.String("endpoint", lc.m.EtcdClientEndpoint), zap.Any("endpoint", cli.Endpoints()),
zap.Error(err), zap.Error(err),
) )
return err return err
@ -178,11 +179,11 @@ func (lc *leaseExpireChecker) checkLease(ctx context.Context, expired bool, leas
return nil return nil
} }
func (lc *leaseExpireChecker) check(expired bool, leases map[int64]time.Time) error { func check(lg *zap.Logger, cli *clientv3.Client, expired bool, leases map[int64]time.Time) error {
ctx, cancel := context.WithTimeout(context.Background(), leaseExpireCheckerTimeout) ctx, cancel := context.WithTimeout(context.Background(), leaseExpireCheckerTimeout)
defer cancel() defer cancel()
for leaseID := range leases { for leaseID := range leases {
if err := lc.checkLease(ctx, expired, leaseID); err != nil { if err := checkLease(ctx, lg, cli, expired, leaseID); err != nil {
return err return err
} }
} }
@ -190,18 +191,18 @@ func (lc *leaseExpireChecker) check(expired bool, leases map[int64]time.Time) er
} }
// TODO: handle failures from "grpc.WaitForReady(true)" // TODO: handle failures from "grpc.WaitForReady(true)"
func (lc *leaseExpireChecker) getLeaseByID(ctx context.Context, leaseID int64) (*clientv3.LeaseTimeToLiveResponse, error) { func getLeaseByID(ctx context.Context, cli *clientv3.Client, leaseID int64) (*clientv3.LeaseTimeToLiveResponse, error) {
return lc.cli.TimeToLive( return cli.TimeToLive(
ctx, ctx,
clientv3.LeaseID(leaseID), clientv3.LeaseID(leaseID),
clientv3.WithAttachedKeys(), clientv3.WithAttachedKeys(),
) )
} }
func (lc *leaseExpireChecker) hasLeaseExpired(ctx context.Context, leaseID int64) (bool, error) { func hasLeaseExpired(ctx context.Context, lg *zap.Logger, cli *clientv3.Client, leaseID int64) (bool, error) {
// keep retrying until lease's state is known or ctx is being canceled // keep retrying until lease's state is known or ctx is being canceled
for ctx.Err() == nil { for ctx.Err() == nil {
resp, err := lc.getLeaseByID(ctx, leaseID) resp, err := getLeaseByID(ctx, cli, leaseID)
if err != nil { if err != nil {
// for ~v3.1 compatibilities // for ~v3.1 compatibilities
if rpctypes.Error(err) == rpctypes.ErrLeaseNotFound { if rpctypes.Error(err) == rpctypes.ErrLeaseNotFound {
@ -210,9 +211,9 @@ func (lc *leaseExpireChecker) hasLeaseExpired(ctx context.Context, leaseID int64
} else { } else {
return resp.TTL == -1, nil return resp.TTL == -1, nil
} }
lc.lg.Warn( lg.Warn(
"hasLeaseExpired getLeaseByID failed", "hasLeaseExpired getLeaseByID failed",
zap.String("endpoint", lc.m.EtcdClientEndpoint), zap.Any("endpoint", cli.Endpoints()),
zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)),
zap.Error(err), zap.Error(err),
) )
@ -223,12 +224,12 @@ func (lc *leaseExpireChecker) hasLeaseExpired(ctx context.Context, leaseID int64
// The keys attached to the lease has the format of "<leaseID>_<idx>" where idx is the ordering key creation // The keys attached to the lease has the format of "<leaseID>_<idx>" where idx is the ordering key creation
// Since the format of keys contains about leaseID, finding keys base on "<leaseID>" prefix // Since the format of keys contains about leaseID, finding keys base on "<leaseID>" prefix
// determines whether the attached keys for a given leaseID has been deleted or not // determines whether the attached keys for a given leaseID has been deleted or not
func (lc *leaseExpireChecker) hasKeysAttachedToLeaseExpired(ctx context.Context, leaseID int64) (bool, error) { func hasKeysAttachedToLeaseExpired(ctx context.Context, lg *zap.Logger, cli *clientv3.Client, leaseID int64) (bool, error) {
resp, err := lc.cli.Get(ctx, fmt.Sprintf("%d", leaseID), clientv3.WithPrefix()) resp, err := cli.Get(ctx, fmt.Sprintf("%d", leaseID), clientv3.WithPrefix())
if err != nil { if err != nil {
lc.lg.Warn( lg.Warn(
"hasKeysAttachedToLeaseExpired failed", "hasKeysAttachedToLeaseExpired failed",
zap.String("endpoint", lc.m.EtcdClientEndpoint), zap.Any("endpoint", cli.Endpoints()),
zap.String("lease-id", fmt.Sprintf("%016x", leaseID)), zap.String("lease-id", fmt.Sprintf("%016x", leaseID)),
zap.Error(err), zap.Error(err),
) )

77
tests/functional/tester/checker_short_ttl_lease_expire.go Normal file
View File

@ -0,0 +1,77 @@
// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tester
import (
"fmt"
"time"
"go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/tests/v3/functional/rpcpb"
"go.uber.org/zap"
"google.golang.org/grpc"
)
type shortTTLLeaseExpireChecker struct {
ctype rpcpb.Checker
lg *zap.Logger
m *rpcpb.Member
ls *leaseStresser
cli *clientv3.Client
}
func newShortTTLLeaseExpireChecker(ls *leaseStresser) Checker {
return &shortTTLLeaseExpireChecker{
ctype: rpcpb.Checker_SHORT_TTL_LEASE_EXPIRE,
lg: ls.lg,
m: ls.m,
ls: ls,
}
}
func (lc *shortTTLLeaseExpireChecker) Type() rpcpb.Checker {
return lc.ctype
}
func (lc *shortTTLLeaseExpireChecker) EtcdClientEndpoints() []string {
return []string{lc.m.EtcdClientEndpoint}
}
func (lc *shortTTLLeaseExpireChecker) Check() error {
if lc.ls == nil {
return nil
}
if lc.ls != nil && lc.ls.alivedLeasesWithShortTTL == nil {
return nil
}
cli, err := lc.m.CreateEtcdClient(grpc.WithBackoffMaxDelay(time.Second))
if err != nil {
return fmt.Errorf("%v (%q)", err, lc.m.EtcdClientEndpoint)
}
defer func() {
if cli != nil {
cli.Close()
}
}()
lc.cli = cli
if err := check(lc.lg, lc.cli, false, lc.ls.alivedLeasesWithShortTTL.leases); err != nil {
lc.lg.Error("failed to check alivedLeasesWithShortTTL", zap.Error(err))
return err
}
lc.lg.Info("check alivedLeasesWithShortTTL succ", zap.Int("num", len(lc.ls.alivedLeasesWithShortTTL.leases)))
return nil
}

14
tests/functional/tester/cluster.go
View File

@ -113,8 +113,6 @@ func NewCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
} }
go clus.serveTesterServer() go clus.serveTesterServer()
clus.updateCases()
clus.rateLimiter = rate.NewLimiter( clus.rateLimiter = rate.NewLimiter(
rate.Limit(int(clus.Tester.StressQPS)), rate.Limit(int(clus.Tester.StressQPS)),
int(clus.Tester.StressQPS), int(clus.Tester.StressQPS),
@ -261,6 +259,13 @@ func (clus *Cluster) updateCases() {
} }
clus.cases = append(clus.cases, clus.cases = append(clus.cases,
fpFailures...) fpFailures...)
case "FAILPOINTS_WITH_DISK_IO_LATENCY":
fpFailures, fperr := failpointDiskIOFailures(clus)
if len(fpFailures) == 0 {
clus.lg.Info("no failpoints found!", zap.Error(fperr))
}
clus.cases = append(clus.cases,
fpFailures...)
} }
} }
} }
@ -322,6 +327,11 @@ func (clus *Cluster) setStresserChecker() {
case "NO_CHECK": case "NO_CHECK":
clus.checkers = append(clus.checkers, newNoChecker()) clus.checkers = append(clus.checkers, newNoChecker())
case "SHORT_TTL_LEASE_EXPIRE":
for _, ls := range lss {
clus.checkers = append(clus.checkers, newShortTTLLeaseExpireChecker(ls))
}
} }
} }
clus.lg.Info("updated stressers") clus.lg.Info("updated stressers")

4
tests/functional/tester/cluster_run.go
View File

@ -33,6 +33,10 @@ const compactQPS = 50000
func (clus *Cluster) Run() { func (clus *Cluster) Run() {
defer printReport() defer printReport()
// updateCases must be executed after etcd is started, because the FAILPOINTS case
// needs to obtain all the failpoints from the etcd member.
clus.updateCases()
if err := fileutil.TouchDirAll(clus.Tester.DataDir); err != nil { if err := fileutil.TouchDirAll(clus.Tester.DataDir); err != nil {
clus.lg.Panic( clus.lg.Panic(
"failed to create test data directory", "failed to create test data directory",

48
tests/functional/tester/stresser_lease.go
View File

@ -48,13 +48,13 @@ type leaseStresser struct {
rateLimiter *rate.Limiter rateLimiter *rate.Limiter
// atomicModifiedKey records the number of keys created and deleted during a test case // atomicModifiedKey records the number of keys created and deleted during a test case
atomicModifiedKey int64 atomicModifiedKey int64
numLeases int numLeases int
keysPerLease int keysPerLease int
aliveLeases *atomicLeases
aliveLeases *atomicLeases alivedLeasesWithShortTTL *atomicLeases
revokedLeases *atomicLeases revokedLeases *atomicLeases
shortLivedLeases *atomicLeases shortLivedLeases *atomicLeases
runWg sync.WaitGroup runWg sync.WaitGroup
aliveWg sync.WaitGroup aliveWg sync.WaitGroup
@ -143,6 +143,7 @@ func (ls *leaseStresser) Stress() error {
ls.revokedLeases = &atomicLeases{leases: make(map[int64]time.Time)} ls.revokedLeases = &atomicLeases{leases: make(map[int64]time.Time)}
ls.shortLivedLeases = &atomicLeases{leases: make(map[int64]time.Time)} ls.shortLivedLeases = &atomicLeases{leases: make(map[int64]time.Time)}
ls.alivedLeasesWithShortTTL = &atomicLeases{leases: make(map[int64]time.Time)}
ls.runWg.Add(1) ls.runWg.Add(1)
go ls.run() go ls.run()
@ -193,9 +194,16 @@ func (ls *leaseStresser) restartKeepAlives() {
ls.keepLeaseAlive(id) ls.keepLeaseAlive(id)
}(leaseID) }(leaseID)
} }
for leaseID := range ls.alivedLeasesWithShortTTL.getLeasesMap() {
ls.aliveWg.Add(1)
go func(id int64) {
ls.keepLeaseAlive(id)
}(leaseID)
}
} }
func (ls *leaseStresser) createLeases() { func (ls *leaseStresser) createLeases() {
ls.createAliveLeasesWithShortTTL()
ls.createAliveLeases() ls.createAliveLeases()
ls.createShortLivedLeases() ls.createShortLivedLeases()
} }
@ -225,6 +233,32 @@ func (ls *leaseStresser) createAliveLeases() {
wg.Wait() wg.Wait()
} }
func (ls *leaseStresser) createAliveLeasesWithShortTTL() {
neededLeases := 2
var wg sync.WaitGroup
for i := 0; i < neededLeases; i++ {
wg.Add(1)
go func() {
defer wg.Done()
leaseID, err := ls.createLeaseWithKeys(defaultTTLShort)
if err != nil {
ls.lg.Debug(
"createLeaseWithKeys failed",
zap.String("endpoint", ls.m.EtcdClientEndpoint),
zap.Error(err),
)
return
}
ls.lg.Warn("createAliveLeasesWithShortTTL", zap.Int64("lease-id", leaseID))
ls.alivedLeasesWithShortTTL.add(leaseID, time.Now())
// keep track of all the keep lease alive goroutines
ls.aliveWg.Add(1)
go ls.keepLeaseAlive(leaseID)
}()
}
wg.Wait()
}
func (ls *leaseStresser) createShortLivedLeases() { func (ls *leaseStresser) createShortLivedLeases() {
// one round of createLeases() might not create all the short lived leases we want due to failures. // one round of createLeases() might not create all the short lived leases we want due to failures.
// thus, we want to create remaining short lived leases in the future round. // thus, we want to create remaining short lived leases in the future round.