mvcc: restore unsynced watchers

In case syncWatchersLoop() starts before Restore() is called,
watchers already added by that moment are moved to s.synced by the loop.
However, there is a broken logic that moves watchers from s.synced
to s.uncyned without setting keyWatchers of the watcherGroup.
Eventually syncWatchers() fails to pickup those watchers from s.unsynced
and no events are sent to the watchers, because newWatcherBatch() called
in the function uses wg.watcherSetByKey() internally that requires
a proper keyWatchers value.

mvcc/watchable_store.go

@@ -267,7 +267,7 @@ func (s *watchableStore) Restore(b backend.Backend) error {
 	}
 
 	for wa := range s.synced.watchers {
-		s.unsynced.watchers.add(wa)
+		s.unsynced.add(wa)
 	}
 	s.synced = newWatcherGroup()
 	return nil

mvcc/watchable_store_test.go

@@ -295,6 +295,8 @@ func TestWatchFutureRev(t *testing.T) {
 }
 
 func TestWatchRestore(t *testing.T) {
+	test := func(delay time.Duration) func(t *testing.T) {
+		return func(t *testing.T) {
 			b, tmpPath := backend.NewDefaultTmpBackend()
 			s := newWatchableStore(b, &lease.FakeLessor{}, nil)
 			defer cleanup(s, b, tmpPath)
@@ -308,7 +310,9 @@ func TestWatchRestore(t *testing.T) {
 			defer cleanup(newStore, newBackend, newPath)
 
 			w := newStore.NewWatchStream()
-	w.Watch(testKey, nil, rev-1)
+			w.Watch(0, testKey, nil, rev-1)
+
+			time.Sleep(delay)
 
 			newStore.Restore(b)
 			select {
@@ -326,6 +330,11 @@ func TestWatchRestore(t *testing.T) {
 				t.Fatal("failed to receive event in 1 second.")
 			}
 		}
+	}
+
+	t.Run("Normal", test(0))
+	t.Run("RunSyncWatchLoopBeforeRestore", test(time.Millisecond*120)) // longer than default waitDuration
+}
 
 // TestWatchBatchUnsynced tests batching on unsynced watchers
 func TestWatchBatchUnsynced(t *testing.T) {