Merge ee36decb1a86f4b2d76967098ca589d5332dadf7 into 517a43ecd132a2089d85f4ebc044728a71d49f6e

This commit is contained in:
Oliver Cai 2025-01-12 07:11:32 +01:00 committed by GitHub
commit d104c8eccd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -169,7 +169,7 @@ class PyTorchInference(Inference):
self.kv_cache = {}
self.hooks = []
def rearrange_kv_cache(self, source_indices):
def rearrange_kv_cache(self, source_indices : List[int]):
if source_indices != list(range(len(source_indices))):
for module in self.kv_modules:
# update the key/value cache to contain the selected sequences