Update README.md

README.md

@@ -35,17 +35,7 @@ base_model = AutoModelForCausalLM.from_pretrained(
 
 # Load tokenizer
 tokenizer = AutoTokenizer.from_pretrained('microsoft/Orca-2-7b')
-tokenizer.add_special_tokens(dict(
-    eos_token=AddedToken("<|im_end|>", single_word=False, lstrip=False, rstrip=False, normalized=True, special=True),
-    unk_token=AddedToken("<unk>", single_word=False, lstrip=False, rstrip=False, normalized=True, special=True),
-    bos_token=AddedToken("<s>", single_word=False, lstrip=False, rstrip=False, normalized=True, special=True),
-    pad_token=AddedToken("</s>", single_word=False, lstrip=False, rstrip=False, normalized=False, special=True),
-))
-tokenizer.add_tokens([AddedToken("<|im_start|>", single_word=False, lstrip=True, rstrip=True, normalized=False)], special_tokens=True)
-tokenizer.additional_special_tokens = ['<unk>', '<s>', '</s>', '<|im_end|>', '<|im_start|>']
-
-model.resize_token_embeddings(len(tokenizer))
-model.config.eos_token_id = tokenizer.eos_token_id
+
 
 # Load PEFT
 model = PeftModel.from_pretrained(base_model, 'Lowenzahn/PathoIE-Orca-2-7B')