
"""simple docstring"""






import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


_lowercase							 =							logging.getLogger(__name__)



def     lowerCAmelCase__ (		)	->List[str]:
						__lowercase						      =					argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."			)
						parser.add_argument(
						    "--dataset_name" ,      type=__magic_name__ ,      default="wikitext" ,      help="Name of the training. Explore datasets at: hf.co/datasets." ,      )
						parser.add_argument(
						    "--dataset_config" ,      type=__magic_name__ ,      default="wikitext-103-raw-v1" ,      help="Configuration name of the dataset."			)
						parser.add_argument(
						    "--batch_size" ,      type=__magic_name__ ,      default=1_0_0_0 ,      help="Batch size during training." ,      )
						parser.add_argument(
						    "--vocab_size" ,      type=__magic_name__ ,      default=1_0_0_4_8 ,      help="Size of the desired vocabulary." ,      )
						parser.add_argument(
						    "--limit" ,      default=__magic_name__ ,      type=__magic_name__ ,      help="Limit the number of shards (used for debugging)." ,      )
						parser.add_argument(
						    "--export_to_hub" ,      action="store_true" ,      )

						__lowercase						      =					parser.parse_args()
						return args



def     lowerCAmelCase__ (		__magic_name__			)	->Union[str, Any]:
						__lowercase						      =					datasets.load_dataset(args.dataset_name ,      args.dataset_config ,      split="train"			)

						if args.limit is not None:
												__lowercase						      =					min(len(__magic_name__			) ,      args.limit			)
												__lowercase						      =					dataset.select(range(__magic_name__			)			)
												logger.info(F'''Limiting the dataset to {args.limit} entries.'''			)

						def batch_iterator():
												for i in range(0 ,      len(__magic_name__			) ,      args.batch_size			):
																		yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
						__lowercase						      =					Tokenizer(Unigram()			)
						__lowercase						      =					normalizers.Sequence([normalizers.Replace("``" ,      "\""			), normalizers.Replace("''" ,      "\""			)]			)
						__lowercase						      =					pre_tokenizers.Metaspace()

						# Prepare the trainer.
						__lowercase						      =					UnigramTrainer(
						    unk_token="<unk>" ,      special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"] ,      vocab_size=args.vocab_size ,      )

						logger.info("Training the tokenizer."			)
						tokenizer.train_from_iterator(batch_iterator() ,      trainer=__magic_name__			)
						logger.info("Tokenizer training complete!"			)

						__lowercase						      =					tokenizer.token_to_id("[CLS]"			)
						__lowercase						      =					tokenizer.token_to_id("[SEP]"			)
						__lowercase						      =					processors.TemplateProcessing(
						    single="[CLS]:0 $A:0 [SEP]:0" ,      pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1" ,      special_tokens=[
						        ("[CLS]", cls_token_id),
						        ("[SEP]", sep_token_id),
						    ] ,      )
						__lowercase						      =					decoders.Metaspace()

						if args.export_to_hub:
												logger.info("Exporting the trained tokenzier to Hub."			)
												__lowercase						      =					AlbertTokenizerFast(tokenizer_object=__magic_name__			)
												new_tokenizer.push_to_hub("unigram-tokenizer-dataset"			)


if __name__ == "__main__":
					_lowercase							 =							parse_args()
					main(args)







"""simple docstring"""






from scipy.constants import g



def     lowerCAmelCase__ (		__magic_name__ ,      __magic_name__			)	->float:
			if mass < 0:
						# handling of negative values of mass
						raise ValueError("The mass of a body cannot be negative"			)
			if height < 0:
						# handling of negative values of height
						raise ValueError("The height above the ground cannot be negative"			)
			return mass * g * height


if __name__ == "__main__":
					from doctest import testmod

					testmod(name='''potential_energy''')





