aehrc
/

cxrmate-ed

anicolson commited on Feb 19

Commit

319482f

verified ·

1 Parent(s): dca7dcb

Upload tokenizer

Files changed (1) hide show

tokenizer_config.json CHANGED Viewed

@@ -58,6 +58,6 @@
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
-  "tokenizer_class": "PreTrainedTokenizerFast",
   "unk_token": "[UNK]"
 }

   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
+  "tokenizer_class": "PreTrainedTokenizer",
   "unk_token": "[UNK]"
 }