dataesr
/

NuExtract-2.0-2B-causalLM

Model card Files Files and versions

ahonestla commited on Sep 9

Commit

ce5b2aa

·

verified ·

1 Parent(s): 8759ed7

fix eos_token

Files changed (1) hide show

tokenizer_config.json +1 -1

tokenizer_config.json CHANGED Viewed

@@ -32,7 +32,7 @@
   ],
   "bos_token": null,
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
   "errors": "replace",
   "extra_special_tokens": {},
   "model_max_length": 32768,

   ],
   "bos_token": null,
   "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
   "errors": "replace",
   "extra_special_tokens": {},
   "model_max_length": 32768,