






'''simple docstring'''







import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


__A      :       int      =	logging.getLogger(__name__)


def 						_UpperCamelCase							(     ):
      __A		   =      argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."							)
      parser.add_argument(
          "--dataset_name"							,			type=snake_case__							,			default="wikitext"							,			help="Name of the training. Explore datasets at: hf.co/datasets."							,			)
      parser.add_argument(
          "--dataset_config"							,			type=snake_case__							,			default="wikitext-103-raw-v1"							,			help="Configuration name of the dataset."							)
      parser.add_argument(
          "--batch_size"							,			type=snake_case__							,			default=1_000							,			help="Batch size during training."							,			)
      parser.add_argument(
          "--vocab_size"							,			type=snake_case__							,			default=10_048							,			help="Size of the desired vocabulary."							,			)
      parser.add_argument(
          "--limit"							,			default=snake_case__							,			type=snake_case__							,			help="Limit the number of shards (used for debugging)."							,			)
      parser.add_argument(
          "--export_to_hub"							,			action="store_true"							,			)

      __A		   =      parser.parse_args()
      return args


def 						_UpperCamelCase							(     snake_case__							):
      __A		   =      datasets.load_dataset(args.dataset_name							,			args.dataset_config							,			split="train"							)

      if args.limit is not None:
            __A		   =      min(len(snake_case__							)							,			args.limit							)
            __A		   =      dataset.select(range(snake_case__							)							)
            logger.info(f"Limiting the dataset to {args.limit} entries."							)

      def batch_iterator():
            for i in range(0							,			len(snake_case__							)							,			args.batch_size							):
                  yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
      __A		   =      Tokenizer(Unigram()							)
      __A		   =      normalizers.Sequence([normalizers.Replace("``"							,			"\""							), normalizers.Replace("''"							,			"\""							)]							)
      __A		   =      pre_tokenizers.Metaspace()

      # Prepare the trainer.
      __A		   =      UnigramTrainer(
          unk_token="<unk>"							,			special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"]							,			vocab_size=args.vocab_size							,			)

      logger.info("Training the tokenizer."							)
      tokenizer.train_from_iterator(batch_iterator()							,			trainer=snake_case__							)
      logger.info("Tokenizer training complete!"							)

      __A		   =      tokenizer.token_to_id("[CLS]"							)
      __A		   =      tokenizer.token_to_id("[SEP]"							)
      __A		   =      processors.TemplateProcessing(
          single="[CLS]:0 $A:0 [SEP]:0"							,			pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"							,			special_tokens=[
              ("[CLS]", cls_token_id),
              ("[SEP]", sep_token_id),
          ]							,			)
      __A		   =      decoders.Metaspace()

      if args.export_to_hub:
            logger.info("Exporting the trained tokenzier to Hub."							)
            __A		   =      AlbertTokenizerFast(tokenizer_object=snake_case__							)
            new_tokenizer.push_to_hub("unigram-tokenizer-dataset"							)


if __name__ == "__main__":
       __A      :       Tuple      =	parse_args()
       main(args)










'''simple docstring'''







def 						_UpperCamelCase							(     snake_case__							,			snake_case__							):
      if n < 0:
            return 0
      # table[i] represents the number of ways to get to amount i
      __A		   =      [0] * (n + 1)

      # There is exactly 1 way to get to zero(You pick no coins).
      __A		   =      1

      # Pick all coins one by one and update table[] values
      # after the index greater than or equal to the value of the
      # picked coin
      for coin_val in s:
            for j in range(snake_case__							,			n + 1							):
                  table[j] += table[j - coin_val]

      return table[n]


if __name__ == "__main__":
       import doctest

       doctest.testmod()


