| from huggingface_hub import login | |
| from tools import Tokenizer | |
| from tools import ChatFormat | |
| from cfg import LLAMA_SIZE_STR | |
| your_token = "" | |
| login(your_token) | |
| from huggingface_hub import hf_hub_download | |
| tokenizer_file_path = hf_hub_download( | |
| repo_id=f"meta-llama/Llama-3.2-{LLAMA_SIZE_STR}-Instruct", | |
| filename="original/tokenizer.model", | |
| local_dir=f"Llama-3.2-{LLAMA_SIZE_STR}-Instruct" | |
| ) | |
| tokenizer = Tokenizer(tokenizer_file_path) | |
| chat_tokenizer = ChatFormat(tokenizer) | |