




'''simple docstring'''


import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


lowerCAmelCase_			:     int							    =     logging.getLogger(__name__)



def    _SCREAMING_SNAKE_CASE    (			):




     """simple docstring"""


     a_    :       int    	=     argparse.ArgumentParser(description="""Train a unigram tokenizer on the wikitext dataset."""							)
     parser.add_argument(
         """--dataset_name"""   ,						type=UpperCamelCase__   ,						default="""wikitext"""   ,						help="""Name of the training. Explore datasets at: hf.co/datasets."""   ,						)
     parser.add_argument(
         """--dataset_config"""   ,						type=UpperCamelCase__   ,						default="""wikitext-103-raw-v1"""   ,						help="""Configuration name of the dataset."""							)
     parser.add_argument(
         """--batch_size"""   ,						type=UpperCamelCase__   ,						default=1000   ,						help="""Batch size during training."""   ,						)
     parser.add_argument(
         """--vocab_size"""   ,						type=UpperCamelCase__   ,						default=1_0048   ,						help="""Size of the desired vocabulary."""   ,						)
     parser.add_argument(
         """--limit"""   ,						default=UpperCamelCase__   ,						type=UpperCamelCase__   ,						help="""Limit the number of shards (used for debugging)."""   ,						)
     parser.add_argument(
         """--export_to_hub"""   ,						action="""store_true"""   ,						)

     a_    :       Dict    	=     parser.parse_args()
     return args



def    _SCREAMING_SNAKE_CASE    (			UpperCamelCase__						:				str							):




     """simple docstring"""


     a_    :       int    	=     datasets.load_dataset(args.dataset_name   ,						args.dataset_config   ,						split="""train"""							)

     if args.limit is not None:
          a_    :       Tuple    	=     min(len(UpperCamelCase__							)   ,						args.limit							)
          a_    :       Optional[Any]    	=     dataset.select(range(UpperCamelCase__							)							)
          logger.info(F"Limiting the dataset to {args.limit} entries."							)

     def batch_iterator():
          for i in range(0   ,						len(UpperCamelCase__							)   ,						args.batch_size							):
               yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
     a_    :       int    	=     Tokenizer(Unigram()							)
     a_    :       Optional[Any]    	=     normalizers.Sequence([normalizers.Replace("""``"""   ,						"""\""""							), normalizers.Replace("""''"""   ,						"""\""""							)]							)
     a_    :       Any    	=     pre_tokenizers.Metaspace()

     # Prepare the trainer.
     a_    :       Optional[int]    	=     UnigramTrainer(
         unk_token="""<unk>"""   ,						special_tokens=["""[CLS]""", """[SEP]""", """<unk>""", """<pad>""", """[MASK]"""]   ,						vocab_size=args.vocab_size   ,						)

     logger.info("""Training the tokenizer."""							)
     tokenizer.train_from_iterator(batch_iterator()   ,						trainer=UpperCamelCase__							)
     logger.info("""Tokenizer training complete!"""							)

     a_    :       int    	=     tokenizer.token_to_id("""[CLS]"""							)
     a_    :       Dict    	=     tokenizer.token_to_id("""[SEP]"""							)
     a_    :       Optional[Any]    	=     processors.TemplateProcessing(
         single="""[CLS]:0 $A:0 [SEP]:0"""   ,						pair="""[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"""   ,						special_tokens=[
             ("""[CLS]""", cls_token_id),
             ("""[SEP]""", sep_token_id),
         ]   ,						)
     a_    :       Tuple    	=     decoders.Metaspace()

     if args.export_to_hub:
          logger.info("""Exporting the trained tokenzier to Hub."""							)
          a_    :       Tuple    	=     AlbertTokenizerFast(tokenizer_object=UpperCamelCase__							)
          new_tokenizer.push_to_hub("""unigram-tokenizer-dataset"""							)


if __name__ == "__main__":
 lowerCAmelCase_			:     List[str]							    =     parse_args()
 main(args)






'''simple docstring'''


import shutil

import requests



def    _SCREAMING_SNAKE_CASE    (			UpperCamelCase__						:				str   ,						UpperCamelCase__						:				bool = False   ,						UpperCamelCase__						:				str = "."							):




     """simple docstring"""


     a_    :       Any    	=     """https://api.nasa.gov/planetary/apod"""
     return requests.get(UpperCamelCase__   ,						params={"""api_key""": api_key}							).json()



def    _SCREAMING_SNAKE_CASE    (			UpperCamelCase__						:				str   ,						UpperCamelCase__						:				str = "."							):




     """simple docstring"""


     a_    :       Tuple    	=     get_apod_data(UpperCamelCase__							)
     a_    :       Tuple    	=     apod_data["""url"""]
     a_    :       Any    	=     img_url.split("""/"""							)[-1]
     a_    :       Tuple    	=     requests.get(UpperCamelCase__   ,						stream=UpperCamelCase__							)

     with open(F"{path}/{img_name}"   ,						"""wb+"""							) as img_file:
          shutil.copyfileobj(response.raw   ,						UpperCamelCase__							)
     del response
     return apod_data



def    _SCREAMING_SNAKE_CASE    (			UpperCamelCase__						:				str							):




     """simple docstring"""


     a_    :       str    	=     """https://images-api.nasa.gov/search"""
     return requests.get(UpperCamelCase__   ,						params={"""q""": query}							).json()


if __name__ == "__main__":
 print(save_apod('YOUR API KEY'))
 lowerCAmelCase_			:     int							    =     get_archive_data('apollo 2011')['collection']['items']
 print(apollo_aaaa_items[0]['data'][0]['description'])
