diff --git a/tests/robustness/validate/validate.go b/tests/robustness/validate/validate.go index b37049039..a1aa4ddb6 100644 --- a/tests/robustness/validate/validate.go +++ b/tests/robustness/validate/validate.go @@ -44,10 +44,16 @@ func ValidateAndReturnVisualize(t *testing.T, lg *zap.Logger, cfg Config, report eventHistory, err := mergeWatchEventHistory(reports) if err != nil { t.Errorf("Failed merging watch history to create event history, err: %s", err) - validateWatch(t, lg, cfg, reports, nil) + err = validateWatch(lg, cfg, reports, nil) + if err != nil { + t.Errorf("Failed validating watch history, err: %s", err) + } return visualize } - validateWatch(t, lg, cfg, reports, eventHistory) + err = validateWatch(lg, cfg, reports, eventHistory) + if err != nil { + t.Errorf("Failed validating watch history, err: %s", err) + } validateSerializableOperations(t, lg, patchedOperations, eventHistory) return visualize } diff --git a/tests/robustness/validate/validate_test.go b/tests/robustness/validate/validate_test.go index 24b56052f..45d53be65 100644 --- a/tests/robustness/validate/validate_test.go +++ b/tests/robustness/validate/validate_test.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//nolint:unparam package validate import ( @@ -60,250 +61,1361 @@ func TestDataReports(t *testing.T) { func TestValidateWatch(t *testing.T) { tcs := []struct { - name string - reports []report.ClientReport + name string + config Config + reports []report.ClientReport + eventHistory []model.PersistedEvent + expectError string }{ { - name: "earlier event after bookmark in separate request", + name: "Ordered, Unique - ordered unique events in one response - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("b", "2", 3, true), + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Ordered, Unique - unique ordered events in separate response - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + }, + }, + { + Events: []model.WatchEvent{ + putWatchEvent("b", "2", 3, true), + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Ordered - unordered events in one response - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("b", "2", 3, true), + putWatchEvent("a", "1", 2, true), + }, + }, + }, + }, + }, + }, + }, + expectError: errBrokeOrdered.Error(), + }, + { + name: "Ordered - unordered events in separate response - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("b", "2", 3, true), + }, + }, + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + }, + }, + }, + }, + }, + }, + }, + expectError: errBrokeOrdered.Error(), + }, + { + name: "Ordered - unordered events in separate watch - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("b", "2", 3, true), + }, + }, + }, + }, + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Unique - duplicated events in one response - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("a", "2", 2, true), + }, + }, + }, + }, + }, + }, + }, + expectError: errBrokeUnique.Error(), + }, + { + name: "Unique - duplicated events in separate responses - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + }, + }, + { + Events: []model.WatchEvent{ + putWatchEvent("a", "2", 2, true), + }, + }, + }, + }, + }, + }, + }, + expectError: errBrokeUnique.Error(), + }, + { + name: "Unique - duplicated events in watch requests - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + }, + }, + }, + }, + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "2", 2, true), + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Unique, Atomic - duplicated revision in one response - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("b", "2", 2, true), + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Unique - duplicated revision in separate watch request - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + }, + }, + }, + }, + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Unique revision - duplicated revision in one response - fail", + config: Config{ExpectRevisionUnique: true}, + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("b", "2", 2, true), + }, + }, + }, + }, + }, + }, + }, + expectError: errBrokeUnique.Error(), + }, + { + name: "Atomic - duplicated revision in one response - fail", + config: Config{ExpectRevisionUnique: true}, + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("b", "2", 2, true), + }, + }, + }, + }, + }, + }, + }, + expectError: errBrokeUnique.Error(), + }, + { + name: "Atomic - revision in separate responses - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + }, + }, + { + Events: []model.WatchEvent{ + putWatchEvent("b", "2", 2, true), + }, + }, + }, + }, + }, + }, + }, + expectError: errBrokeAtomic.Error(), + }, + { + name: "Resumable, Reliable, Bookmarkable - all events with bookmark - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ - Key: "a", - Revision: 2, + Key: "", + WithPrefix: true, }, Responses: []model.WatchResponse{ { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("b", "2", 3, true), + putWatchEvent("c", "3", 4, true), + }, + }, + { + Revision: 4, IsProgressNotify: true, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + }, + { + name: "Resumable, Reliable, Bookmarkable - empty events without revision - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "", + WithPrefix: true, + }, + Responses: []model.WatchResponse{ + { + Revision: 4, + IsProgressNotify: true, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + }, + { + name: "Resumable, Reliable, Bookmarkable - empty events with watch revision - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Revision: 2, + Key: "", + WithPrefix: true, + }, + Responses: []model.WatchResponse{ + { + Revision: 4, + IsProgressNotify: true, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + // TODO: Should fail as it should guarantee that all events between requested revision and bookmark are delivered + expectError: "", + }, + { + name: "Bookmarkable - revision non decreasing - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "", + WithPrefix: true, + }, + Responses: []model.WatchResponse{ + { + Revision: 1, + IsProgressNotify: true, + }, + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + }, + }, + { + Revision: 2, + IsProgressNotify: true, + }, + { + Events: []model.WatchEvent{ + putWatchEvent("b", "2", 3, true), + }, + }, + { Revision: 3, + IsProgressNotify: true, + }, + { + Revision: 3, + IsProgressNotify: true, }, - }, - }, - { - Request: model.WatchRequest{ - Key: "a", - Revision: 2, - }, - Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ - { - PersistedEvent: model.PersistedEvent{ - Event: model.Event{ - Type: model.PutOperation, - Key: "a", - Value: model.ValueOrHash{ - Value: "99", - }, - }, - Revision: 2, - IsCreate: true, - }, - }, + putWatchEvent("c", "3", 4, true), }, - Revision: 3, + }, + { + Revision: 4, + IsProgressNotify: true, }, }, }, }, }, }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, }, { - name: "earlier event after in separate request", + name: "Bookmarkable - event precedes progress - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ - Key: "a", - Revision: 3, + Key: "", + WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ - { - PersistedEvent: model.PersistedEvent{ - Event: model.Event{ - Type: model.PutOperation, - Key: "a", - Value: model.ValueOrHash{ - Value: "100", - }, - }, - Revision: 3, - }, - }, + putWatchEvent("a", "1", 2, true), + putWatchEvent("b", "2", 3, true), + putWatchEvent("c", "3", 4, true), }, - Revision: 3, }, - }, - }, - { - Request: model.WatchRequest{ - Key: "a", - Revision: 2, - }, - Responses: []model.WatchResponse{ { - Events: []model.WatchEvent{ - { - PersistedEvent: model.PersistedEvent{ - Event: model.Event{ - Type: model.PutOperation, - Key: "a", - Value: model.ValueOrHash{ - Value: "99", - }, - }, - Revision: 2, - IsCreate: true, - }, - }, - }, - Revision: 3, + Revision: 3, + IsProgressNotify: true, }, }, }, }, }, }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + // TODO: This should fail as bookmark lowered revision + expectError: "", }, { - name: "duplicated event between two separate requests", + name: "Bookmarkable - progress precedes event - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ - Key: "a", - Revision: 2, + Key: "", + WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ - { - PersistedEvent: model.PersistedEvent{ - Event: model.Event{ - Type: model.PutOperation, - Key: "a", - Value: model.ValueOrHash{ - Value: "100", - }, - }, - Revision: 2, - IsCreate: true, - }, - }, + putWatchEvent("a", "1", 2, true), + putWatchEvent("b", "2", 3, true), }, - Revision: 2, }, - }, - }, - { - Request: model.WatchRequest{ - Key: "a", - Revision: 2, - }, - Responses: []model.WatchResponse{ + { + Revision: 4, + IsProgressNotify: true, + }, { Events: []model.WatchEvent{ - { - PersistedEvent: model.PersistedEvent{ - Event: model.Event{ - Type: model.PutOperation, - Key: "a", - Value: model.ValueOrHash{ - Value: "100", - }, - }, - Revision: 2, - IsCreate: true, - }, - }, + putWatchEvent("c", "3", 4, true), }, - Revision: 2, }, }, }, }, }, }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + expectError: errBrokeBookmarkable.Error(), }, { - name: "create event and update event in separate requests both with PrevKV()", + name: "Bookmarkable - missing event before bookmark - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "", + WithPrefix: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("b", "2", 3, true), + }, + }, + { + Revision: 4, + IsProgressNotify: true, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + // TODO: This should fail as there is a missing event before bookmark + expectError: "", + }, + { + name: "Bookmarkable - missing event matching watch before bookmark - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "a", + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + }, + }, + { + Revision: 4, + IsProgressNotify: true, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("a", "2", 3, false), + putPersistedEvent("c", "3", 4, true), + }, + // TODO: This should fail as there is a missing event before bookmark + expectError: "", + }, + { + name: "Bookmarkable - missing event matching watch with prefix before bookmark - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", - Revision: 2, - WithPrevKV: true, + WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ - { - PersistedEvent: model.PersistedEvent{ - Event: model.Event{ - Type: model.PutOperation, - Key: "a", - Value: model.ValueOrHash{ - Value: "100", - }, - }, - Revision: 2, - IsCreate: true, - }, - }, + putWatchEvent("aa", "1", 2, true), }, - Revision: 2, }, - }, - }, - { - Request: model.WatchRequest{ - Key: "a", - Revision: 3, - }, - Responses: []model.WatchResponse{ { - Events: []model.WatchEvent{ - { - PersistedEvent: model.PersistedEvent{ - Event: model.Event{ - Type: model.PutOperation, - Key: "a", - Value: model.ValueOrHash{ - Value: "101", - }, - }, - Revision: 3, - }, - PrevValue: &model.ValueRevision{ - Value: model.ToValueOrHash("100"), - ModRevision: 2, - }, - }, - }, - Revision: 3, + Revision: 4, + IsProgressNotify: true, }, }, }, }, }, }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("aa", "1", 2, true), + putPersistedEvent("ab", "2", 3, true), + putPersistedEvent("cc", "3", 4, true), + }, + // TODO: This should fail as there is a missing event before bookmark + expectError: "", + }, + { + name: "Reliable - all events history - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "", + WithPrefix: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("b", "2", 3, true), + putWatchEvent("c", "3", 4, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + expectError: "", + }, + { + name: "Reliable - missing middle event - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "", + WithPrefix: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("c", "3", 4, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + expectError: errBrokeReliable.Error(), + }, + { + name: "Reliable - middle event doesn't match request - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "a", + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("a", "3", 4, false), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("ab", "2", 3, true), + putPersistedEvent("a", "3", 4, false), + }, + }, + { + name: "Reliable - middle event doesn't match request with prefix - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "a", + WithPrefix: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("aa", "1", 2, true), + putWatchEvent("ac", "3", 4, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("aa", "1", 2, true), + putPersistedEvent("bb", "2", 3, true), + putPersistedEvent("ac", "3", 4, true), + }, + }, + { + name: "Reliable, Resumable - missing first event - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "", + WithPrefix: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("b", "2", 3, true), + putWatchEvent("c", "3", 4, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + // TODO: Should pass as watch with revision 0 might start from any revision. + expectError: errBrokeResumable.Error(), + }, + { + name: "Reliable - missing last event - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "", + WithPrefix: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("b", "2", 3, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + }, + { + name: "Resumable - watch revision from middle event - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "", + WithPrefix: true, + Revision: 3, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("b", "2", 3, true), + putWatchEvent("c", "3", 4, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + }, + { + name: "Resumable - watch key from middle event - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "b", + Revision: 2, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("b", "2", 3, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + }, + { + name: "Resumable - watch key with prefix from middle event - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "b", + Revision: 2, + WithPrefix: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("bb", "2", 3, true), + putWatchEvent("bc", "3", 4, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("bb", "2", 3, true), + putPersistedEvent("bc", "3", 4, true), + }, + }, + { + name: "Resumable - missing first matching event - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "", + WithPrefix: true, + Revision: 3, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("c", "3", 4, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("c", "3", 4, true), + }, + expectError: errBrokeResumable.Error(), + }, + { + name: "Resumable - missing first matching event with prefix - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "b", + Revision: 2, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("b", "3", 4, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("b", "3", 4, true), + }, + expectError: errBrokeResumable.Error(), + }, + { + name: "Resumable - missing first matching event with prefix - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "b", + WithPrefix: true, + Revision: 2, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("bc", "3", 4, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("bb", "2", 3, true), + putPersistedEvent("bc", "3", 4, true), + }, + expectError: errBrokeResumable.Error(), + }, + { + name: "IsCreate - correct IsCreate values - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("a", "2", 3, false), + deleteWatchEvent("a", 4), + putWatchEvent("a", "4", 5, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("a", "2", 3, false), + deletePersistedEvent("a", 4), + putPersistedEvent("a", "4", 5, true), + }, + }, + { + name: "IsCreate - second put marked as created - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("a", "2", 3, true), + deleteWatchEvent("a", 4), + putWatchEvent("a", "4", 5, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("a", "2", 3, false), + deletePersistedEvent("a", 4), + putPersistedEvent("a", "4", 5, true), + }, + expectError: errBrokeIsCreate.Error(), + }, + { + name: "IsCreate - put after delete marked as not created - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("a", "2", 3, false), + deleteWatchEvent("a", 4), + putWatchEvent("a", "4", 5, false), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("a", "2", 3, false), + deletePersistedEvent("a", 4), + putPersistedEvent("a", "4", 5, true), + }, + expectError: errBrokeIsCreate.Error(), + }, + { + name: "PrevKV - no previous values - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + WithPrevKV: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("a", "2", 3, false), + deleteWatchEvent("a", 4), + putWatchEvent("a", "4", 5, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("a", "2", 3, false), + deletePersistedEvent("a", 4), + putPersistedEvent("a", "4", 5, true), + }, + }, + { + name: "PrevKV - all previous values - pass", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + WithPrevKV: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEventWithPrevKV("a", "2", 3, false, "1", 2), + deleteWatchEventWithPrevKV("a", 4, "2", 3), + putWatchEvent("a", "4", 5, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("a", "2", 3, false), + deletePersistedEvent("a", 4), + putPersistedEvent("a", "4", 5, true), + }, + }, + { + name: "PrevKV - mismatch value on put - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + WithPrevKV: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEventWithPrevKV("a", "2", 3, false, "2", 2), + deleteWatchEventWithPrevKV("a", 4, "2", 3), + putWatchEvent("a", "4", 5, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("a", "2", 3, false), + deletePersistedEvent("a", 4), + putPersistedEvent("a", "4", 5, true), + }, + expectError: errBrokePrevKV.Error(), + }, + { + name: "PrevKV - mismatch revision on put - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + WithPrevKV: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEventWithPrevKV("a", "2", 3, false, "1", 3), + deleteWatchEventWithPrevKV("a", 4, "2", 3), + putWatchEvent("a", "4", 5, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("a", "2", 3, false), + deletePersistedEvent("a", 4), + putPersistedEvent("a", "4", 5, true), + }, + expectError: errBrokePrevKV.Error(), + }, + { + name: "PrevKV - mismatch value on delete - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + WithPrevKV: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEventWithPrevKV("a", "2", 3, false, "1", 2), + deleteWatchEventWithPrevKV("a", 4, "1", 3), + putWatchEvent("a", "4", 5, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("a", "2", 3, false), + deletePersistedEvent("a", 4), + putPersistedEvent("a", "4", 5, true), + }, + expectError: errBrokePrevKV.Error(), + }, + { + name: "PrevKV - mismatch revision on delete - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + WithPrevKV: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEventWithPrevKV("a", "2", 3, false, "1", 2), + deleteWatchEventWithPrevKV("a", 4, "2", 2), + putWatchEvent("a", "4", 5, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("a", "2", 3, false), + deletePersistedEvent("a", 4), + putPersistedEvent("a", "4", 5, true), + }, + expectError: errBrokePrevKV.Error(), + }, + { + name: "Filter - event not matching the watch - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "a", + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("a", "1", 2, true), + putWatchEvent("b", "2", 3, true), + putWatchEvent("a", "3", 4, false), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("a", "1", 2, true), + putPersistedEvent("b", "2", 3, true), + putPersistedEvent("a", "3", 4, false), + }, + // TODO: Test should fail as there is an event not matching selector + expectError: "", + }, + { + name: "Filter - event not matching the watch with prefix - fail", + reports: []report.ClientReport{ + { + Watch: []model.WatchOperation{ + { + Request: model.WatchRequest{ + Key: "a", + WithPrefix: true, + }, + Responses: []model.WatchResponse{ + { + Events: []model.WatchEvent{ + putWatchEvent("aa", "1", 2, true), + putWatchEvent("bb", "2", 3, true), + putWatchEvent("ac", "3", 4, true), + }, + }, + }, + }, + }, + }, + }, + eventHistory: []model.PersistedEvent{ + putPersistedEvent("aa", "1", 2, true), + putPersistedEvent("bb", "2", 3, true), + putPersistedEvent("ac", "3", 4, true), + }, + // TODO: Test should fail as there is an event not matching selector + expectError: "", }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { - eventHistory, err := mergeWatchEventHistory(tc.reports) + err := validateWatch(zaptest.NewLogger(t), tc.config, tc.reports, tc.eventHistory) + var errStr string if err != nil { - t.Fatal(err) + errStr = err.Error() + } + if errStr != tc.expectError { + t.Errorf("validateWatch(...), got: %q, want: %q", err, tc.expectError) } - validateWatch(t, zaptest.NewLogger(t), Config{ExpectRevisionUnique: true}, tc.reports, eventHistory) }) } } + +func putWatchEvent(key, value string, rev int64, isCreate bool) model.WatchEvent { + return model.WatchEvent{ + PersistedEvent: putPersistedEvent(key, value, rev, isCreate), + } +} + +func deleteWatchEvent(key string, rev int64) model.WatchEvent { + return model.WatchEvent{ + PersistedEvent: deletePersistedEvent(key, rev), + } +} + +func putWatchEventWithPrevKV(key, value string, rev int64, isCreate bool, prevValue string, modRev int64) model.WatchEvent { + return model.WatchEvent{ + PersistedEvent: putPersistedEvent(key, value, rev, isCreate), + PrevValue: &model.ValueRevision{ + Value: model.ToValueOrHash(prevValue), + ModRevision: modRev, + }, + } +} + +func deleteWatchEventWithPrevKV(key string, rev int64, prevValue string, modRev int64) model.WatchEvent { + return model.WatchEvent{ + PersistedEvent: deletePersistedEvent(key, rev), + PrevValue: &model.ValueRevision{ + Value: model.ToValueOrHash(prevValue), + ModRevision: modRev, + }, + } +} + +func putPersistedEvent(key, value string, rev int64, isCreate bool) model.PersistedEvent { + return model.PersistedEvent{ + Event: model.Event{ + Type: model.PutOperation, + Key: key, + Value: model.ToValueOrHash(value), + }, + Revision: rev, + IsCreate: isCreate, + } +} + +func deletePersistedEvent(key string, rev int64) model.PersistedEvent { + return model.PersistedEvent{ + Event: model.Event{ + Type: model.DeleteOperation, + Key: key, + }, + Revision: rev, + } +} diff --git a/tests/robustness/validate/watch.go b/tests/robustness/validate/watch.go index eeb00c6cb..a43e527f5 100644 --- a/tests/robustness/validate/watch.go +++ b/tests/robustness/validate/watch.go @@ -15,7 +15,7 @@ package validate import ( - "testing" + "errors" "go.uber.org/zap" @@ -23,30 +23,67 @@ import ( "go.etcd.io/etcd/tests/v3/robustness/report" ) -func validateWatch(t *testing.T, lg *zap.Logger, cfg Config, reports []report.ClientReport, eventHistory []model.PersistedEvent) { +var ( + errBrokeBookmarkable = errors.New("broke Bookmarkable - Progress notification events guarantee that all events up to a revision have been already delivered") + errBrokeOrdered = errors.New("broke Ordered - events are ordered by revision; an event will never appear on a watch if it precedes an event in time that has already been posted") + errBrokeUnique = errors.New("broke Unique - an event will never appear on a watch twice") + errBrokeAtomic = errors.New("broke Atomic - a list of events is guaranteed to encompass complete revisions; updates in the same revision over multiple keys will not be split over several lists of events") + errBrokeReliable = errors.New("broke Reliable - a sequence of events will never drop any subsequence of events; if there are events ordered in time as a < b < c, then if the watch receives events a and c, it is guaranteed to receive b") + errBrokeResumable = errors.New("broke Resumable - A broken watch can be resumed by establishing a new watch starting after the last revision received in a watch event before the break, so long as the revision is in the history window") + errBrokePrevKV = errors.New("incorrect event prevValue") + errBrokeIsCreate = errors.New("incorrect event IsCreate") +) + +func validateWatch(lg *zap.Logger, cfg Config, reports []report.ClientReport, eventHistory []model.PersistedEvent) error { lg.Info("Validating watch") // Validate etcd watch properties defined in https://etcd.io/docs/v3.6/learning/api_guarantees/#watch-apis for _, r := range reports { - validateOrdered(t, r) - validateUnique(t, cfg.ExpectRevisionUnique, r) - validateAtomic(t, r) - validateBookmarkable(t, r) + err := validateOrdered(lg, r) + if err != nil { + return err + } + err = validateUnique(lg, cfg.ExpectRevisionUnique, r) + if err != nil { + return err + } + err = validateAtomic(lg, r) + if err != nil { + return err + } + err = validateBookmarkable(lg, r) + if err != nil { + return err + } if eventHistory != nil { - validateReliable(t, eventHistory, r) - validateResumable(t, eventHistory, r) - validatePrevKV(t, r, eventHistory) - validateCreateEvent(t, r, eventHistory) + err = validateReliable(lg, eventHistory, r) + if err != nil { + return err + } + err = validateResumable(lg, eventHistory, r) + if err != nil { + return err + } + err = validatePrevKV(lg, r, eventHistory) + if err != nil { + return err + } + err = validateEventIsCreate(lg, r, eventHistory) + if err != nil { + return err + } } } + return nil } -func validateBookmarkable(t *testing.T, report report.ClientReport) { +func validateBookmarkable(lg *zap.Logger, report report.ClientReport) (err error) { for _, op := range report.Watch { var lastProgressNotifyRevision int64 for _, resp := range op.Responses { for _, event := range resp.Events { if event.Revision <= lastProgressNotifyRevision { - t.Errorf("Broke watch guarantee: Bookmarkable - Progress notification events guarantee that all events up to a revision have been already delivered, eventRevision: %d, progressNotifyRevision: %d", event.Revision, lastProgressNotifyRevision) + lg.Error("Broke watch guarantee", zap.String("guarantee", "bookmarkable"), zap.Int("client", report.ClientID), zap.Int64("revision", event.Revision)) + err = errBrokeBookmarkable } } if resp.IsProgressNotify { @@ -54,23 +91,26 @@ func validateBookmarkable(t *testing.T, report report.ClientReport) { } } } + return err } -func validateOrdered(t *testing.T, report report.ClientReport) { +func validateOrdered(lg *zap.Logger, report report.ClientReport) (err error) { for _, op := range report.Watch { var lastEventRevision int64 = 1 for _, resp := range op.Responses { for _, event := range resp.Events { if event.Revision < lastEventRevision { - t.Errorf("Broke watch guarantee: Ordered - events are ordered by revision; an event will never appear on a watch if it precedes an event in time that has already been posted, lastRevision: %d, currentRevision: %d, client: %d", lastEventRevision, event.Revision, report.ClientID) + lg.Error("Broke watch guarantee", zap.String("guarantee", "ordered"), zap.Int("client", report.ClientID), zap.Int64("revision", event.Revision)) + err = errBrokeOrdered } lastEventRevision = event.Revision } } } + return err } -func validateUnique(t *testing.T, expectUniqueRevision bool, report report.ClientReport) { +func validateUnique(lg *zap.Logger, expectUniqueRevision bool, report report.ClientReport) (err error) { for _, op := range report.Watch { uniqueOperations := map[any]struct{}{} for _, resp := range op.Responses { @@ -85,29 +125,33 @@ func validateUnique(t *testing.T, expectUniqueRevision bool, report report.Clien }{event.Revision, event.Key} } if _, found := uniqueOperations[key]; found { - t.Errorf("Broke watch guarantee: Unique - an event will never appear on a watch twice, key: %q, revision: %d, client: %d", event.Key, event.Revision, report.ClientID) + lg.Error("Broke watch guarantee", zap.String("guarantee", "unique"), zap.Int("client", report.ClientID), zap.String("key", event.Key), zap.Int64("revision", event.Revision)) + err = errBrokeUnique } uniqueOperations[key] = struct{}{} } } } + return err } -func validateAtomic(t *testing.T, report report.ClientReport) { +func validateAtomic(lg *zap.Logger, report report.ClientReport) (err error) { for _, op := range report.Watch { var lastEventRevision int64 = 1 for _, resp := range op.Responses { if len(resp.Events) > 0 { if resp.Events[0].Revision == lastEventRevision { - t.Errorf("Broke watch guarantee: Atomic - a list of events is guaranteed to encompass complete revisions; updates in the same revision over multiple keys will not be split over several lists of events, previousListEventRevision: %d, currentListEventRevision: %d, client: %d", lastEventRevision, resp.Events[0].Revision, report.ClientID) + lg.Error("Broke watch guarantee", zap.String("guarantee", "atomic"), zap.Int("client", report.ClientID), zap.Int64("revision", resp.Events[0].Revision)) + err = errBrokeAtomic } lastEventRevision = resp.Events[len(resp.Events)-1].Revision } } } + return err } -func validateReliable(t *testing.T, events []model.PersistedEvent, report report.ClientReport) { +func validateReliable(lg *zap.Logger, events []model.PersistedEvent, report report.ClientReport) (err error) { for _, op := range report.Watch { index := 0 revision := firstRevision(op) @@ -120,15 +164,17 @@ func validateReliable(t *testing.T, events []model.PersistedEvent, report report for _, resp := range op.Responses { for _, event := range resp.Events { if events[index].Match(op.Request) && events[index] != event.PersistedEvent { - t.Errorf("Broke watch guarantee: Reliable - a sequence of events will never drop any subsequence of events; if there are events ordered in time as a < b < c, then if the watch receives events a and c, it is guaranteed to receive b, event missing: %+v, got: %+v", events[index], event) + lg.Error("Broke watch guarantee", zap.String("guarantee", "reliable"), zap.Int("client", report.ClientID), zap.Any("missing-event", events[index])) + err = errBrokeReliable } index++ } } } + return err } -func validateResumable(t *testing.T, events []model.PersistedEvent, report report.ClientReport) { +func validateResumable(lg *zap.Logger, events []model.PersistedEvent, report report.ClientReport) (err error) { for _, op := range report.Watch { index := 0 revision := op.Request.Revision @@ -141,14 +187,16 @@ func validateResumable(t *testing.T, events []model.PersistedEvent, report repor firstEvent := firstWatchEvent(op) // If watch is resumable, first event it gets should the first event that happened after the requested revision. if firstEvent != nil && events[index] != firstEvent.PersistedEvent { - t.Errorf("Resumable - A broken watch can be resumed by establishing a new watch starting after the last revision received in a watch event before the break, so long as the revision is in the history window, watch request: %+v, event missing: %+v, got: %+v", op.Request, events[index], *firstEvent) + lg.Error("Broke watch guarantee", zap.String("guarantee", "resumable"), zap.Int("client", report.ClientID), zap.Any("request", op.Request), zap.Any("got-event", *firstEvent), zap.Any("want-event", events[index])) + err = errBrokeResumable } } + return err } // validatePrevKV ensures that a watch response (if configured with WithPrevKV()) returns // the appropriate response. -func validatePrevKV(t *testing.T, report report.ClientReport, history []model.PersistedEvent) { +func validatePrevKV(lg *zap.Logger, report report.ClientReport, history []model.PersistedEvent) (err error) { replay := model.NewReplay(history) for _, op := range report.Watch { if !op.Request.WithPrevKV { @@ -157,10 +205,7 @@ func validatePrevKV(t *testing.T, report report.ClientReport, history []model.Pe for _, resp := range op.Responses { for _, event := range resp.Events { // Get state state just before the current event. - state, err := replay.StateForRevision(event.Revision - 1) - if err != nil { - t.Error(err) - } + state, _ := replay.StateForRevision(event.Revision - 1) // TODO(MadhavJivrajani): check if compaction has been run as part // of failpoint injection. If compaction has run, prevKV can be nil // even if it is not a create event. @@ -175,31 +220,32 @@ func validatePrevKV(t *testing.T, report report.ClientReport, history []model.Pe // We allow PrevValue to be nil since in the face of compaction, etcd does not // guarantee its presence. if event.PrevValue != nil && *event.PrevValue != state.KeyValues[event.Key] { - t.Errorf("PrevKV - PrevValue doesn't match previous value under the key %s, got: %+v, want: %+v", event.Key, *event.PrevValue, state.KeyValues[event.Key]) + lg.Error("Incorrect event prevValue field", zap.Int("client", report.ClientID), zap.Any("event", event), zap.Any("previousValue", state.KeyValues[event.Key])) + err = errBrokePrevKV } } } } + return err } -func validateCreateEvent(t *testing.T, report report.ClientReport, history []model.PersistedEvent) { +func validateEventIsCreate(lg *zap.Logger, report report.ClientReport, history []model.PersistedEvent) (err error) { replay := model.NewReplay(history) for _, op := range report.Watch { for _, resp := range op.Responses { for _, event := range resp.Events { // Get state state just before the current event. - state, err := replay.StateForRevision(event.Revision - 1) - if err != nil { - t.Error(err) - } + state, _ := replay.StateForRevision(event.Revision - 1) // A create event will not have an entry in our history and a non-create // event *should* have an entry in our history. if _, prevKeyExists := state.KeyValues[event.Key]; event.IsCreate == prevKeyExists { - t.Errorf("CreateEvent - unexpected event ecountered, create event should not be in event history and update/delete event should be, event already exists: %t, is create event: %t, event: %+v", prevKeyExists, event.IsCreate, event) + lg.Error("Incorrect event IsCreate field", zap.Int("client", report.ClientID), zap.Any("event", event)) + err = errBrokeIsCreate } } } } + return err } func firstRevision(op model.WatchOperation) int64 {