fix token suppression (#123)

This commit is contained in:
VulumeCode 2022-09-26 13:35:21 +02:00 committed by GitHub
parent ead77fab97
commit 520796a34c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -537,8 +537,7 @@ class DecodingTask:
elif suppress_tokens is None or len(suppress_tokens) == 0:
suppress_tokens = [] # interpret empty string as an empty list
else:
assert isinstance(self.options.suppress_tokens, list), "suppress_tokens must be a list"
suppress_tokens = self.options.suppress_tokens
assert isinstance(suppress_tokens, list), "suppress_tokens must be a list"
suppress_tokens.extend(
[self.tokenizer.sot, self.tokenizer.sot_prev, self.tokenizer.sot_lm]