lfu commited on
Commit
ab6cc91
·
verified ·
1 Parent(s): c52f808

Update tokenization_kimi.py

Browse files

Pass added_tokens_decoder to the base class so that the non-special added tokens will be saved during `save_pretrained`.

Files changed (1) hide show
  1. tokenization_kimi.py +1 -0
tokenization_kimi.py CHANGED
@@ -151,6 +151,7 @@ class TikTokenTokenizer(PreTrainedTokenizer):
151
  unk_token=unk_token,
152
  pad_token=pad_token,
153
  additional_special_tokens=additional_special_tokens,
 
154
  **kwargs,
155
  )
156
  self.all_special_ids_set = set(self.all_special_ids)
 
151
  unk_token=unk_token,
152
  pad_token=pad_token,
153
  additional_special_tokens=additional_special_tokens,
154
+ added_tokens_decoder=added_tokens_decoder,
155
  **kwargs,
156
  )
157
  self.all_special_ids_set = set(self.all_special_ids)