diff --git a/server/etcdserver/api/rafthttp/transport.go b/server/etcdserver/api/rafthttp/transport.go
index 940e8473b..fa3011cb3 100644
--- a/server/etcdserver/api/rafthttp/transport.go
+++ b/server/etcdserver/api/rafthttp/transport.go
@@ -339,24 +339,30 @@ func (t *Transport) RemoveAllPeers() {
 
 // the caller of this function must have the peers mutex.
 func (t *Transport) removePeer(id types.ID) {
-	if peer, ok := t.peers[id]; ok {
+	// etcd may remove a member again on startup due to WAL files replaying.
+	peer, ok := t.peers[id]
+	if ok {
 		peer.stop()
-	} else {
-		if t.Logger != nil {
-			t.Logger.Panic("unexpected removal of unknown remote peer", zap.String("remote-peer-id", id.String()))
-		}
+		delete(t.peers, id)
+		delete(t.LeaderStats.Followers, id.String())
+		t.pipelineProber.Remove(id.String())
+		t.streamProber.Remove(id.String())
 	}
-	delete(t.peers, id)
-	delete(t.LeaderStats.Followers, id.String())
-	t.pipelineProber.Remove(id.String())
-	t.streamProber.Remove(id.String())
 
 	if t.Logger != nil {
-		t.Logger.Info(
-			"removed remote peer",
-			zap.String("local-member-id", t.ID.String()),
-			zap.String("removed-remote-peer-id", id.String()),
-		)
+		if ok {
+			t.Logger.Info(
+				"removed remote peer",
+				zap.String("local-member-id", t.ID.String()),
+				zap.String("removed-remote-peer-id", id.String()),
+			)
+		} else {
+			t.Logger.Warn(
+				"skipped removing already removed peer",
+				zap.String("local-member-id", t.ID.String()),
+				zap.String("removed-remote-peer-id", id.String()),
+			)
+		}
 	}
 }