




import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


SCREAMING_SNAKE_CASE__ :   Union[str, Any]						=     logging.getLogger(__name__)

def   A   (   )			->				Union[str, Any]:

       '''simple docstring'''





       lowerCamelCase     :   Tuple    				=      argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."					)
       parser.add_argument(
           "--dataset_name"				,type=_SCREAMING_SNAKE_CASE				,default="wikitext"				,help="Name of the training. Explore datasets at: hf.co/datasets."				,)
       parser.add_argument(
           "--dataset_config"				,type=_SCREAMING_SNAKE_CASE				,default="wikitext-103-raw-v1"				,help="Configuration name of the dataset."					)
       parser.add_argument(
           "--batch_size"				,type=_SCREAMING_SNAKE_CASE				,default=1000				,help="Batch size during training."				,)
       parser.add_argument(
           "--vocab_size"				,type=_SCREAMING_SNAKE_CASE				,default=1_0048				,help="Size of the desired vocabulary."				,)
       parser.add_argument(
           "--limit"				,default=_SCREAMING_SNAKE_CASE				,type=_SCREAMING_SNAKE_CASE				,help="Limit the number of shards (used for debugging)."				,)
       parser.add_argument(
           "--export_to_hub"				,action="store_true"				,)

       lowerCamelCase     :   str    				=      parser.parse_args()
       return args

def   A   (   _SCREAMING_SNAKE_CASE					)			->				List[str]:

       '''simple docstring'''





       lowerCamelCase     :   int    				=      datasets.load_dataset(args.dataset_name				,args.dataset_config				,split="train"					)

       if args.limit is not None:
              lowerCamelCase     :   Any    				=      min(len(_SCREAMING_SNAKE_CASE					)				,args.limit					)
              lowerCamelCase     :   Dict    				=      dataset.select(range(_SCREAMING_SNAKE_CASE					)					)
              logger.info(f'''Limiting the dataset to {args.limit} entries.'''					)

       def batch_iterator():
              for i in range(0				,len(_SCREAMING_SNAKE_CASE					)				,args.batch_size					):
                     yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
       lowerCamelCase     :   List[str]    				=      Tokenizer(Unigram()					)
       lowerCamelCase     :   Tuple    				=      normalizers.Sequence([normalizers.Replace("``"				,"\""					), normalizers.Replace("''"				,"\""					)]					)
       lowerCamelCase     :   Tuple    				=      pre_tokenizers.Metaspace()

       # Prepare the trainer.
       lowerCamelCase     :   Optional[Any]    				=      UnigramTrainer(
           unk_token="<unk>"				,special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"]				,vocab_size=args.vocab_size				,)

       logger.info("Training the tokenizer."					)
       tokenizer.train_from_iterator(batch_iterator()				,trainer=_SCREAMING_SNAKE_CASE					)
       logger.info("Tokenizer training complete!"					)

       lowerCamelCase     :   Union[str, Any]    				=      tokenizer.token_to_id("[CLS]"					)
       lowerCamelCase     :   Tuple    				=      tokenizer.token_to_id("[SEP]"					)
       lowerCamelCase     :   Union[str, Any]    				=      processors.TemplateProcessing(
           single="[CLS]:0 $A:0 [SEP]:0"				,pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"				,special_tokens=[
               ("[CLS]", cls_token_id),
               ("[SEP]", sep_token_id),
           ]				,)
       lowerCamelCase     :   str    				=      decoders.Metaspace()

       if args.export_to_hub:
              logger.info("Exporting the trained tokenzier to Hub."					)
              lowerCamelCase     :   Any    				=      AlbertTokenizerFast(tokenizer_object=_SCREAMING_SNAKE_CASE					)
              new_tokenizer.push_to_hub("unigram-tokenizer-dataset"					)


if __name__ == "__main__":
       SCREAMING_SNAKE_CASE__ :   List[str]						=     parse_args()
       main(args)












import subprocess
from typing import Union

import numpy as np
import requests

from ..utils import add_end_docstrings, is_torch_available, is_torchaudio_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_torch_available():
       from ..models.auto.modeling_auto import MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING

SCREAMING_SNAKE_CASE__ :   Union[str, Any]						=     logging.get_logger(__name__)

def   A   (   _SCREAMING_SNAKE_CASE				,_SCREAMING_SNAKE_CASE					)			->				np.array:
       lowerCamelCase     :   Optional[Any]    				=      f'''{sampling_rate}'''
       lowerCamelCase     :   Optional[Any]    				=      "1"
       lowerCamelCase     :   Tuple    				=      "f32le"
       lowerCamelCase     :   Any    				=      [
           "ffmpeg",
           "-i",
           "pipe:0",
           "-ac",
           ac,
           "-ar",
           ar,
           "-f",
           format_for_conversion,
           "-hide_banner",
           "-loglevel",
           "quiet",
           "pipe:1",
       ]

       try:
              lowerCamelCase     :   Optional[int]    				=      subprocess.Popen(_SCREAMING_SNAKE_CASE				,stdin=subprocess.PIPE				,stdout=subprocess.PIPE					)
       except FileNotFoundError:
              raise ValueError("ffmpeg was not found but is required to load audio files from filename"					)
       lowerCamelCase     :   Optional[int]    				=      ffmpeg_process.communicate(_SCREAMING_SNAKE_CASE					)
       lowerCamelCase     :   Union[str, Any]    				=      output_stream[0]

       lowerCamelCase     :   Dict    				=      np.frombuffer(_SCREAMING_SNAKE_CASE				,np.floataa					)
       if audio.shape[0] == 0:
              raise ValueError("Malformed soundfile"					)
       return audio




@add_end_docstrings(lowerCAmelCase__  )
class 	UpperCamelCase__			(lowerCAmelCase__  ):

    '''simple docstring'''



    def __init__(					self  ,						*UpperCamelCase__  ,						**UpperCamelCase__  )  ->   Dict:
           # Default, might be overriden by the model.config.
           lowerCamelCase     :   Union[str, Any]    				=      5
           super().__init__(*UpperCamelCase__  ,						**UpperCamelCase__  )

           if self.framework != "pt":
                  raise ValueError(F'''The {self.__class__} is only available in PyTorch.'''  )

           self.check_model_type(UpperCamelCase__  )







    def __call__(					self  ,						UpperCamelCase__  ,						**UpperCamelCase__  ,						)  ->   Optional[int]:
           return super().__call__(UpperCamelCase__  ,						**UpperCamelCase__  )







    def 					_lowercase  (					self  ,						UpperCamelCase__=None  ,						**UpperCamelCase__  )  ->   str:
           # No parameters on this pipeline right now
           lowerCamelCase     :   int    				=      {}
           if top_k is not None:
                  if top_k > self.model.config.num_labels:
                         lowerCamelCase     :   Any    				=      self.model.config.num_labels
                  lowerCamelCase     :   Any    				=      top_k
           return {}, {}, postprocess_params







    def 					_lowercase  (					self  ,						UpperCamelCase__  )  ->   Dict:
           if isinstance(UpperCamelCase__  ,						UpperCamelCase__  ):
                  if inputs.startswith("http://"  ) or inputs.startswith("https://"  ):
                         # We need to actually check for a real protocol, otherwise it's impossible to use a local file
                         # like http_huggingface_co.png
                         lowerCamelCase     :   str    				=      requests.get(UpperCamelCase__  ).content
                  else:
                         with open(UpperCamelCase__  ,						"rb"  ) as f:
                                lowerCamelCase     :   int    				=      f.read()

           if isinstance(UpperCamelCase__  ,						UpperCamelCase__  ):
                  lowerCamelCase     :   str    				=      ffmpeg_read(UpperCamelCase__  ,						self.feature_extractor.sampling_rate  )

           if isinstance(UpperCamelCase__  ,						UpperCamelCase__  ):
                  # Accepting `"array"` which is the key defined in `datasets` for
                  # better integration
                  if not ("sampling_rate" in inputs and ("raw" in inputs or "array" in inputs)):
                         raise ValueError(
                             "When passing a dictionary to AudioClassificationPipeline, the dict needs to contain a "
                             "\"raw\" key containing the numpy array representing the audio and a \"sampling_rate\" key, "
                             "containing the sampling_rate associated with that array"  )

                  lowerCamelCase     :   Optional[int]    				=      inputs.pop("raw"  ,						UpperCamelCase__  )
                  if _inputs is None:
                         # Remove path which will not be used from `datasets`.
                         inputs.pop("path"  ,						UpperCamelCase__  )
                         lowerCamelCase     :   List[str]    				=      inputs.pop("array"  ,						UpperCamelCase__  )
                  lowerCamelCase     :   str    				=      inputs.pop("sampling_rate"  )
                  lowerCamelCase     :   Union[str, Any]    				=      _inputs
                  if in_sampling_rate != self.feature_extractor.sampling_rate:
                         import torch

                         if is_torchaudio_available():
                                from torchaudio import functional as F
                         else:
                                raise ImportError(
                                    "torchaudio is required to resample audio samples in AudioClassificationPipeline. "
                                    "The torchaudio package can be installed through: `pip install torchaudio`."  )

                         lowerCamelCase     :   Tuple    				=      F.resample(
                             torch.from_numpy(UpperCamelCase__  )  ,						UpperCamelCase__  ,						self.feature_extractor.sampling_rate  ).numpy()

           if not isinstance(UpperCamelCase__  ,						np.ndarray  ):
                  raise ValueError("We expect a numpy ndarray as input"  )
           if len(inputs.shape  ) != 1:
                  raise ValueError("We expect a single channel audio input for AudioClassificationPipeline"  )

           lowerCamelCase     :   Optional[int]    				=      self.feature_extractor(
               UpperCamelCase__  ,						sampling_rate=self.feature_extractor.sampling_rate  ,						return_tensors="pt"  )
           return processed







    def 					_lowercase  (					self  ,						UpperCamelCase__  )  ->   str:
           lowerCamelCase     :   Optional[Any]    				=      self.model(**UpperCamelCase__  )
           return model_outputs







    def 					_lowercase  (					self  ,						UpperCamelCase__  ,						UpperCamelCase__=5  )  ->   Tuple:
           lowerCamelCase     :   Tuple    				=      model_outputs.logits[0].softmax(-1  )
           lowerCamelCase				,   lowerCamelCase     :   str    				=      probs.topk(UpperCamelCase__  )

           lowerCamelCase     :   str    				=      scores.tolist()
           lowerCamelCase     :   Optional[int]    				=      ids.tolist()

           lowerCamelCase     :   Optional[int]    				=      [{"score": score, "label": self.model.config.idalabel[_id]} for score, _id in zip(UpperCamelCase__  ,						UpperCamelCase__  )]

           return labels






