Merge ee36decb1a86f4b2d76967098ca589d5332dadf7 into c0d2f624c09dc18e709e37c2ad90c039a4eb72a2

This commit is contained in:
Oliver Cai 2025-06-27 02:27:44 +00:00 committed by GitHub
commit ee60e2e148
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -169,7 +169,7 @@ class PyTorchInference(Inference):
self.kv_cache = {}
self.hooks = []
def rearrange_kv_cache(self, source_indices):
def rearrange_kv_cache(self, source_indices : List[int]):
if source_indices != list(range(len(source_indices))):
for module in self.kv_modules:
# update the key/value cache to contain the selected sequences