


"""simple docstring"""



import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


lowercase_							 =       logging.getLogger(__name__)
def     UpperCAmelCase    (					)	->   List[Any]:


	"""simple docstring"""




	lowerCAmelCase_			       = argparse.ArgumentParser(description='''Train a unigram tokenizer on the wikitext dataset.'''  )
	parser.add_argument(
	    '''--dataset_name''' ,      type=_lowercase ,      default='''wikitext''' ,      help='''Name of the training. Explore datasets at: hf.co/datasets.''' ,      )
	parser.add_argument(
	    '''--dataset_config''' ,      type=_lowercase ,      default='''wikitext-103-raw-v1''' ,      help='''Configuration name of the dataset.'''  )
	parser.add_argument(
	    '''--batch_size''' ,      type=_lowercase ,      default=1_0_0_0 ,      help='''Batch size during training.''' ,      )
	parser.add_argument(
	    '''--vocab_size''' ,      type=_lowercase ,      default=1_0_0_4_8 ,      help='''Size of the desired vocabulary.''' ,      )
	parser.add_argument(
	    '''--limit''' ,      default=_lowercase ,      type=_lowercase ,      help='''Limit the number of shards (used for debugging).''' ,      )
	parser.add_argument(
	    '''--export_to_hub''' ,      action='''store_true''' ,      )

	lowerCAmelCase_			       = parser.parse_args()
	return args
def     UpperCAmelCase    (					_lowercase   :      List[str]  )	->   List[str]:


	"""simple docstring"""




	lowerCAmelCase_			       = datasets.load_dataset(args.dataset_name ,      args.dataset_config ,      split='''train'''  )

	if args.limit is not None:
		lowerCAmelCase_			       = min(len(_lowercase  ) ,      args.limit  )
		lowerCAmelCase_			       = dataset.select(range(_lowercase  )  )
		logger.info(F"""Limiting the dataset to {args.limit} entries."""  )

	def batch_iterator():
		for i in range(0 ,      len(_lowercase  ) ,      args.batch_size  ):
			yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
	lowerCAmelCase_			       = Tokenizer(Unigram()  )
	lowerCAmelCase_			       = normalizers.Sequence([normalizers.Replace('''``''' ,      '''"'''  ), normalizers.Replace('''\'\'''' ,      '''"'''  )]  )
	lowerCAmelCase_			       = pre_tokenizers.Metaspace()

	# Prepare the trainer.
	lowerCAmelCase_			       = UnigramTrainer(
	    unk_token='''<unk>''' ,      special_tokens=['''[CLS]''', '''[SEP]''', '''<unk>''', '''<pad>''', '''[MASK]'''] ,      vocab_size=args.vocab_size ,      )

	logger.info('''Training the tokenizer.'''  )
	tokenizer.train_from_iterator(batch_iterator() ,      trainer=_lowercase  )
	logger.info('''Tokenizer training complete!'''  )

	lowerCAmelCase_			       = tokenizer.token_to_id('''[CLS]'''  )
	lowerCAmelCase_			       = tokenizer.token_to_id('''[SEP]'''  )
	lowerCAmelCase_			       = processors.TemplateProcessing(
	    single='''[CLS]:0 $A:0 [SEP]:0''' ,      pair='''[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1''' ,      special_tokens=[
	        ('''[CLS]''', cls_token_id),
	        ('''[SEP]''', sep_token_id),
	    ] ,      )
	lowerCAmelCase_			       = decoders.Metaspace()

	if args.export_to_hub:
		logger.info('''Exporting the trained tokenzier to Hub.'''  )
		lowerCAmelCase_			       = AlbertTokenizerFast(tokenizer_object=_lowercase  )
		new_tokenizer.push_to_hub('''unigram-tokenizer-dataset'''  )


if __name__ == "__main__":
		lowercase_							 =       parse_args()
		main(args)



"""simple docstring"""



def     UpperCAmelCase    (					_lowercase   :      list[int]  )	->   int:


	"""simple docstring"""




	lowerCAmelCase_			       = len(_lowercase  )
	if n in (0, 1):
		return 0
	return sum(query * (n - i - 1) for i, query in enumerate(sorted(_lowercase  )  )  )


if __name__ == "__main__":
		import doctest

		doctest.testmod()