




import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


UpperCAmelCase__						:       Dict							=    logging.getLogger(__name__)


def 		_lowerCamelCase				(					)			->		int:





  """simple docstring"""





  _lowercase: List[Any]						     =    argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'							)
  parser.add_argument(
      '--dataset_name'					,     type=UpperCamelCase					,     default='wikitext'					,     help='Name of the training. Explore datasets at: hf.co/datasets.'					,     )
  parser.add_argument(
      '--dataset_config'					,     type=UpperCamelCase					,     default='wikitext-103-raw-v1'					,     help='Configuration name of the dataset.'							)
  parser.add_argument(
      '--batch_size'					,     type=UpperCamelCase					,     default=1_000					,     help='Batch size during training.'					,     )
  parser.add_argument(
      '--vocab_size'					,     type=UpperCamelCase					,     default=10_048					,     help='Size of the desired vocabulary.'					,     )
  parser.add_argument(
      '--limit'					,     default=UpperCamelCase					,     type=UpperCamelCase					,     help='Limit the number of shards (used for debugging).'					,     )
  parser.add_argument(
      '--export_to_hub'					,     action='store_true'					,     )

  _lowercase: Optional[int]						     =    parser.parse_args()
  return args


def 		_lowerCamelCase				(					UpperCamelCase							)			->		List[str]:





  """simple docstring"""





  _lowercase: Union[str, Any]						     =    datasets.load_dataset(args.dataset_name					,     args.dataset_config					,     split='train'							)

  if args.limit is not None:
    _lowercase: Any						     =    min(len(UpperCamelCase							)					,     args.limit							)
    _lowercase: Any						     =    dataset.select(range(UpperCamelCase							)							)
    logger.info(F'Limiting the dataset to {args.limit} entries.'							)

  def batch_iterator():
    for i in range(0					,     len(UpperCamelCase							)					,     args.batch_size							):
      yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
  _lowercase: List[Any]						     =    Tokenizer(Unigram()							)
  _lowercase: Optional[Any]						     =    normalizers.Sequence([normalizers.Replace('``'					,     '"'							), normalizers.Replace('\'\''					,     '"'							)]							)
  _lowercase: Any						     =    pre_tokenizers.Metaspace()

  # Prepare the trainer.
  _lowercase: str						     =    UnigramTrainer(
      unk_token='<unk>'					,     special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]']					,     vocab_size=args.vocab_size					,     )

  logger.info('Training the tokenizer.'							)
  tokenizer.train_from_iterator(batch_iterator()					,     trainer=UpperCamelCase							)
  logger.info('Tokenizer training complete!'							)

  _lowercase: Optional[Any]						     =    tokenizer.token_to_id('[CLS]'							)
  _lowercase: List[str]						     =    tokenizer.token_to_id('[SEP]'							)
  _lowercase: str						     =    processors.TemplateProcessing(
      single='[CLS]:0 $A:0 [SEP]:0'					,     pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'					,     special_tokens=[
          ('[CLS]', cls_token_id),
          ('[SEP]', sep_token_id),
      ]					,     )
  _lowercase: str						     =    decoders.Metaspace()

  if args.export_to_hub:
    logger.info('Exporting the trained tokenzier to Hub.'							)
    _lowercase: Optional[Any]						     =    AlbertTokenizerFast(tokenizer_object=UpperCamelCase							)
    new_tokenizer.push_to_hub('unigram-tokenizer-dataset'							)


if __name__ == "__main__":
   UpperCAmelCase__						:       Tuple							=    parse_args()
   main(args)






from cva import destroyAllWindows, imread, imshow, waitKey


def 		_lowerCamelCase				(					UpperCamelCase					,     UpperCamelCase							)			->		List[str]:





  """simple docstring"""





  _lowercase     ,					_lowercase: Tuple						     =    img.shape[0], img.shape[1]

  def to_grayscale(UpperCamelCase					,     UpperCamelCase					,     UpperCamelCase							):
    return 0.2_126 * red + 0.587 * green + 0.114 * blue

  def normalize(UpperCamelCase							):
    return min(UpperCamelCase					,     255							)

  for i in range(UpperCamelCase							):
    for j in range(UpperCamelCase							):
      _lowercase: Any						     =    int(to_grayscale(*img[i][j]							)							)
      _lowercase: str						     =    [
          normalize(UpperCamelCase							),
          normalize(greyscale + factor							),
          normalize(greyscale + 2 * factor							),
      ]

  return img


if __name__ == "__main__":
   # read original image
   UpperCAmelCase__						:       Optional[Any]							=    {
       percentage: imread('image_data/lena.jpg', 1) for percentage in (10, 20, 30, 40)
   }

   for percentage, img in images.items():
      make_sepia(img, percentage)

   for percentage, img in images.items():
      imshow(f'''Original image with sepia (factor: {percentage})''', img)

   waitKey(0)
   destroyAllWindows()
