|
|
import json
|
|
|
import os
|
|
|
from typing import List, Optional
|
|
|
from transformers import PreTrainedTokenizer
|
|
|
|
|
|
|
|
|
class CenturioTokenizer(PreTrainedTokenizer):
|
|
|
vocab_files_names = {"vocab_file": "centurio_vocab.json"}
|
|
|
model_input_names = ["input_ids", "attention_mask"]
|
|
|
|
|
|
def __init__(
|
|
|
self,
|
|
|
vocab_file=None,
|
|
|
unk_token="<unk>",
|
|
|
bos_token="<s>",
|
|
|
eos_token="</s>",
|
|
|
pad_token="<pad>",
|
|
|
sep_token="<sep>",
|
|
|
cls_token="<cls>",
|
|
|
mask_token="<mask>",
|
|
|
space_token="▁",
|
|
|
**kwargs
|
|
|
):
|
|
|
self.space_token = space_token
|
|
|
self._vocab = {}
|
|
|
self._inv_vocab = {}
|
|
|
|
|
|
super().__init__(
|
|
|
unk_token=unk_token,
|
|
|
bos_token=bos_token,
|
|
|
eos_token=eos_token,
|
|
|
pad_token=pad_token,
|
|
|
sep_token=sep_token,
|
|
|
cls_token=cls_token,
|
|
|
mask_token=mask_token,
|
|
|
**kwargs
|
|
|
)
|
|
|
|
|
|
if vocab_file is not None:
|
|
|
self._load_vocab(vocab_file)
|
|
|
else:
|
|
|
self._build_default_vocab()
|
|
|
|
|
|
def _build_default_vocab(self):
|
|
|
special_tokens = [
|
|
|
self.unk_token, self.bos_token, self.eos_token,
|
|
|
self.pad_token, self.sep_token, self.cls_token,
|
|
|
self.mask_token, self.space_token
|
|
|
]
|
|
|
self._vocab = {token: i for i, token in enumerate(special_tokens)}
|
|
|
self._inv_vocab = {i: token for token, i in self._vocab.items()}
|
|
|
|
|
|
def _load_vocab(self, vocab_file):
|
|
|
with open(vocab_file, "r", encoding="utf-8") as f:
|
|
|
self._vocab = json.load(f)
|
|
|
self._inv_vocab = {v: k for k, v in self._vocab.items()}
|
|
|
|
|
|
def get_vocab(self):
|
|
|
return self._vocab.copy()
|
|
|
|
|
|
@property
|
|
|
def vocab_size(self):
|
|
|
return len(self._vocab)
|
|
|
|
|
|
def _tokenize(self, text: str) -> List[str]:
|
|
|
text = text.replace(" ", self.space_token)
|
|
|
tokens = []
|
|
|
current = ""
|
|
|
for ch in text:
|
|
|
if ch.isalnum() or ch in "абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ":
|
|
|
current += ch
|
|
|
else:
|
|
|
if current:
|
|
|
tokens.append(current)
|
|
|
current = ""
|
|
|
tokens.append(ch if ch != self.space_token else self.space_token)
|
|
|
if current:
|
|
|
tokens.append(current)
|
|
|
return tokens
|
|
|
|
|
|
def _convert_token_to_id(self, token: str) -> int:
|
|
|
return self._vocab.get(token, self._vocab.get(self.unk_token, 0))
|
|
|
|
|
|
def _convert_id_to_token(self, index: int) -> str:
|
|
|
return self._inv_vocab.get(index, self.unk_token)
|
|
|
|
|
|
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple:
|
|
|
if not os.path.isdir(save_directory):
|
|
|
os.makedirs(save_directory)
|
|
|
vocab_file = os.path.join(
|
|
|
save_directory,
|
|
|
(filename_prefix + "-" if filename_prefix else "") + "centurio_vocab.json"
|
|
|
)
|
|
|
with open(vocab_file, "w", encoding="utf-8") as f:
|
|
|
json.dump(self._vocab, f, ensure_ascii=False, indent=2)
|
|
|
return (vocab_file,)
|
|
|
|
|
|
def build_vocab_from_corpus(self, corpus: List[str], min_freq: int = 2):
|
|
|
from collections import Counter
|
|
|
token_counter = Counter()
|
|
|
for text in corpus:
|
|
|
tokens = self._tokenize(text)
|
|
|
token_counter.update(tokens)
|
|
|
|
|
|
special_tokens = [
|
|
|
self.unk_token, self.bos_token, self.eos_token,
|
|
|
self.pad_token, self.sep_token, self.cls_token,
|
|
|
self.mask_token, self.space_token
|
|
|
]
|
|
|
new_vocab = {token: i for i, token in enumerate(special_tokens)}
|
|
|
idx = len(new_vocab)
|
|
|
|
|
|
for token, freq in token_counter.items():
|
|
|
if freq >= min_freq and token not in new_vocab:
|
|
|
new_vocab[token] = idx
|
|
|
idx += 1
|
|
|
|
|
|
self._vocab = new_vocab
|
|
|
self._inv_vocab = {v: k for k, v in self._vocab.items()}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
corpus = [
|
|
|
"Привет, как дела!",
|
|
|
"Я учу немецкий язык.",
|
|
|
"Морфемы помогают понять структуру слов."
|
|
|
]
|
|
|
|
|
|
tokenizer = CenturioTokenizer()
|
|
|
tokenizer.build_vocab_from_corpus(corpus, min_freq=1)
|
|
|
tokenizer.save_pretrained("./centurio_model")
|
|
|
|
|
|
for text in corpus:
|
|
|
tokens = tokenizer.tokenize(text)
|
|
|
ids = tokenizer.encode(text)
|
|
|
back = tokenizer.decode(ids)
|
|
|
print(f"\nTEXT : {text}")
|
|
|
print(f"TOKENS : {tokens}")
|
|
|
print(f"IDS : {ids}")
|
|
|
print(f"BACK : {back}")
|
|
|
print(f"VOCAB : {tokenizer.vocab_size}")
|
|
|
|
