Update tokenization_kimi.py
Browse filesPass added_tokens_decoder to the base class so that the non-special added tokens will be saved during `save_pretrained`.
- tokenization_kimi.py +1 -0
tokenization_kimi.py
CHANGED
|
@@ -151,6 +151,7 @@ class TikTokenTokenizer(PreTrainedTokenizer):
|
|
| 151 |
unk_token=unk_token,
|
| 152 |
pad_token=pad_token,
|
| 153 |
additional_special_tokens=additional_special_tokens,
|
|
|
|
| 154 |
**kwargs,
|
| 155 |
)
|
| 156 |
self.all_special_ids_set = set(self.all_special_ids)
|
|
|
|
| 151 |
unk_token=unk_token,
|
| 152 |
pad_token=pad_token,
|
| 153 |
additional_special_tokens=additional_special_tokens,
|
| 154 |
+
added_tokens_decoder=added_tokens_decoder,
|
| 155 |
**kwargs,
|
| 156 |
)
|
| 157 |
self.all_special_ids_set = set(self.all_special_ids)
|