



import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase						  =     logging.get_logger(__name__)
__UpperCAmelCase						  =     {'''vocab_file''': '''spiece.model'''}

__UpperCAmelCase						  =     {
    '''vocab_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
    },
}

__UpperCAmelCase						  =     {
    '''google/fnet-base''': 5_12,
    '''google/fnet-large''': 5_12,
}

__UpperCAmelCase						  =     '''▁'''


class 	lowerCamelCase__ (		_a   ):
      _lowerCAmelCase								= VOCAB_FILES_NAMES
      _lowerCAmelCase								= PRETRAINED_VOCAB_FILES_MAP
      _lowerCAmelCase								= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowerCAmelCase								= ['''input_ids''', '''token_type_ids''']


      def __init__( self						:  Tuple      ,       _a						:  List[str]      ,       _a						:  Any=False      ,       _a						:  List[str]=True      ,       _a						:  str=True      ,       _a						:  Optional[Any]="<unk>"      ,       _a						:  str="[SEP]"      ,       _a						:  Union[str, Any]="<pad>"      ,       _a						:  Dict="[CLS]"      ,       _a						:  int="[MASK]"      ,       _a						:  Optional[Dict[str, Any]] = None      ,       **_a						:  Dict      ,       ):
             # Mask token behave like a normal word, i.e. include the space before it and
             # is included in the raw text, there should be a match in a non-normalized sentence.
             a__:       Dict											=(
                 AddedToken(_a      ,       lstrip=_a      ,       rstrip=_a      ,       normalized=_a			)
                 if isinstance(_a      ,       _a			)
                 else mask_token
             )

             a__:       str											={} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 do_lower_case=_a      ,       remove_space=_a      ,       keep_accents=_a      ,       unk_token=_a      ,       sep_token=_a      ,       pad_token=_a      ,       cls_token=_a      ,       mask_token=_a      ,       sp_model_kwargs=self.sp_model_kwargs      ,       **_a      ,       )

             a__:       List[str]											=do_lower_case
             a__:       List[str]											=remove_space
             a__:       int											=keep_accents
             a__:       Any											=vocab_file

             a__:       Union[str, Any]											=spm.SentencePieceProcessor(**self.sp_model_kwargs			)
             self.sp_model.Load(_a			)


      @property
      def   _lowerCamelCase			( self						:  str			):
             return len(self.sp_model			)


      def   _lowerCamelCase			( self						:  int			):
             a__:       Any											={self.convert_ids_to_tokens(_a			): i for i in range(self.vocab_size			)}
             vocab.update(self.added_tokens_encoder			)
             return vocab


      def __getstate__( self						:  int			):
             a__:       str											=self.__dict__.copy()
             a__:       List[Any]											=None
             return state


      def __setstate__( self						:  Dict      ,       _a						:  List[Any]			):
             a__:       List[Any]											=d

             # for backward compatibility
             if not hasattr(self      ,       "sp_model_kwargs"			):
                    a__:       Any											={}

             a__:       List[str]											=spm.SentencePieceProcessor(**self.sp_model_kwargs			)
             self.sp_model.Load(self.vocab_file			)


      def   _lowerCamelCase			( self						:  List[str]      ,       _a						:  int			):
             if self.remove_space:
                    a__:       List[Any]											=" ".join(inputs.strip().split()			)
             else:
                    a__:       Dict											=inputs
             a__:       Any											=outputs.replace("``"      ,       "\""			).replace("''"      ,       "\""			)

             if not self.keep_accents:
                    a__:       Union[str, Any]											=unicodedata.normalize("NFKD"      ,       _a			)
                    a__:       int											="".join([c for c in outputs if not unicodedata.combining(_a			)]			)
             if self.do_lower_case:
                    a__:       str											=outputs.lower()

             return outputs


      def   _lowerCamelCase			( self						:  Any      ,       _a						:  str			):
             a__:       List[Any]											=self.preprocess_text(_a			)
             a__:       Tuple											=self.sp_model.encode(_a      ,       out_type=_a			)
             a__:       Dict											=[]
             for piece in pieces:
                    if len(_a			) > 1 and piece[-1] == str(","			) and piece[-2].isdigit():
                           a__:       Union[str, Any]											=self.sp_model.EncodeAsPieces(piece[:-1].replace(_a      ,       ""			)			)
                           if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                  if len(cur_pieces[0]			) == 1:
                                         a__:       Any											=cur_pieces[1:]
                                  else:
                                         a__:       Tuple											=cur_pieces[0][1:]
                           cur_pieces.append(piece[-1]			)
                           new_pieces.extend(_a			)
                    else:
                           new_pieces.append(_a			)

             return new_pieces


      def   _lowerCamelCase			( self						:  Tuple      ,       _a						:  Tuple			):
             return self.sp_model.PieceToId(_a			)


      def   _lowerCamelCase			( self						:  Any      ,       _a						:  List[Any]			):
             return self.sp_model.IdToPiece(_a			)


      def   _lowerCamelCase			( self						:  Optional[Any]      ,       _a						:  Tuple			):
             a__:       Tuple											=[]
             a__:       Tuple											=""
             a__:       Union[str, Any]											=False
             for token in tokens:
                    # make sure that special tokens are not decoded using sentencepiece model
                    if token in self.all_special_tokens:
                           if not prev_is_special:
                                  out_string += " "
                           out_string += self.sp_model.decode(_a			) + token
                           a__:       Optional[Any]											=True
                           a__:       Any											=[]
                    else:
                           current_sub_tokens.append(_a			)
                           a__:       int											=False
             out_string += self.sp_model.decode(_a			)
             return out_string.strip()


      def   _lowerCamelCase			( self						:  Optional[Any]      ,       _a						:  List[int]      ,       _a						:  bool = False      ,       _a						:  bool = None      ,       _a						:  bool = True      ,       **_a						:  Optional[int]      ,       ):
             a__:       Optional[Any]											=kwargs.pop("use_source_tokenizer"      ,       _a			)

             a__:       List[str]											=self.convert_ids_to_tokens(_a      ,       skip_special_tokens=_a			)

             # To avoid mixing byte-level and unicode for byte-level BPT
             # we need to build string separately for added tokens and byte-level tokens
             # cf. https://github.com/huggingface/transformers/issues/1133
             a__:       str											=[]
             a__:       int											=[]
             for token in filtered_tokens:
                    if skip_special_tokens and token in self.all_special_ids:
                           continue
                    if token in self.added_tokens_encoder:
                           if current_sub_text:
                                  sub_texts.append(self.convert_tokens_to_string(_a			)			)
                                  a__:       List[str]											=[]
                           sub_texts.append(_a			)
                    else:
                           current_sub_text.append(_a			)
             if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(_a			)			)

             # Mimic the behavior of the Rust tokenizer:
             # No space after <unk>
             if spaces_between_special_tokens:
                    a__:       Any											=re.sub(r"(<unk>) "      ,       r"\1"      ,       " ".join(_a			)			)
             else:
                    a__:       int											="".join(_a			)

             a__:       Optional[int]											=(
                 clean_up_tokenization_spaces
                 if clean_up_tokenization_spaces is not None
                 else self.clean_up_tokenization_spaces
             )
             if clean_up_tokenization_spaces:
                    a__:       Tuple											=self.clean_up_tokenization(_a			)
                    return clean_text
             else:
                    return text


      def   _lowerCamelCase			( self						:  Optional[int]      ,       _a						:  List[int]      ,       _a						:  Optional[List[int]] = None			):
             a__:       List[Any]											=[self.sep_token_id]
             a__:       List[str]											=[self.cls_token_id]
             if token_ids_a is None:
                    return cls + token_ids_a + sep
             return cls + token_ids_a + sep + token_ids_a + sep


      def   _lowerCamelCase			( self						:  Optional[Any]      ,       _a						:  List[int]      ,       _a						:  Optional[List[int]] = None      ,       _a						:  bool = False			):

             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=_a      ,       token_ids_a=_a      ,       already_has_special_tokens=_a			)

             if token_ids_a is not None:
                    return [1] + ([0] * len(_a			)) + [1] + ([0] * len(_a			)) + [1]
             return [1] + ([0] * len(_a			)) + [1]


      def   _lowerCamelCase			( self						:  Tuple      ,       _a						:  List[int]      ,       _a						:  Optional[List[int]] = None			):
             a__:       Dict											=[self.sep_token_id]
             a__:       int											=[self.cls_token_id]

             if token_ids_a is None:
                    return len(cls + token_ids_a + sep			) * [0]
             return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]







      def   _lowerCamelCase			( self						:  Tuple      ,       _a						:  str      ,       _a						:  Optional[str] = None			):
             if not os.path.isdir(_a			):
                    logger.error(F"Vocabulary path ({save_directory}) should be a directory"			)
                    return
             a__:       str											=os.path.join(
                 _a      ,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)

             if os.path.abspath(self.vocab_file			) != os.path.abspath(_a			) and os.path.isfile(self.vocab_file			):
                    copyfile(self.vocab_file      ,       _a			)
             elif not os.path.isfile(self.vocab_file			):
                    with open(_a      ,       "wb"			) as fi:
                           a__:       List[str]											=self.sp_model.serialized_model_proto()
                           fi.write(_a			)

             return (out_vocab_file,)










import subprocess
from typing import Union

import numpy as np
import requests

from ..utils import add_end_docstrings, is_torch_available, is_torchaudio_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_torch_available():
		from ..models.auto.modeling_auto import MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING

__UpperCAmelCase						  =     logging.get_logger(__name__)





def 						__lowerCamelCase       (      __magic_name__	:  bytes     , __magic_name__	:  int						):
							a__:       Optional[int]											=F"{sampling_rate}"
							a__:       Union[str, Any]											="1"
							a__:       List[str]											="f32le"
							a__:       Dict											=[
							    "ffmpeg",
							    "-i",
							    "pipe:0",
							    "-ac",
							    ac,
							    "-ar",
							    ar,
							    "-f",
							    format_for_conversion,
							    "-hide_banner",
							    "-loglevel",
							    "quiet",
							    "pipe:1",
							]

							try:
														a__:       List[Any]											=subprocess.Popen(__magic_name__     , stdin=subprocess.PIPE     , stdout=subprocess.PIPE						)
							except FileNotFoundError:
														raise ValueError("ffmpeg was not found but is required to load audio files from filename"						)
							a__:       int											=ffmpeg_process.communicate(__magic_name__						)
							a__:       List[str]											=output_stream[0]

							a__:       str											=np.frombuffer(__magic_name__     , np.floataa						)
							if audio.shape[0] == 0:
														raise ValueError("Malformed soundfile"						)
							return audio







@add_end_docstrings(_a   )
class 	lowerCamelCase__ (		_a   ):


						def __init__( self						:  Union[str, Any]      ,       *_a						:  Dict      ,       **_a						:  Tuple			):
													# Default, might be overriden by the model.config.
													a__:       List[Any]											=5
													super().__init__(*_a      ,       **_a			)

													if self.framework != "pt":
																				raise ValueError(F"The {self.__class__} is only available in PyTorch."			)

													self.check_model_type(_a			)


						def __call__( self						:  Tuple      ,       _a						:  Union[np.ndarray, bytes, str]      ,       **_a						:  Union[str, Any]      ,       ):
													return super().__call__(_a      ,       **_a			)


						def   _lowerCamelCase			( self						:  int      ,       _a						:  Tuple=None      ,       **_a						:  int			):
													# No parameters on this pipeline right now
													a__:       Optional[int]											={}
													if top_k is not None:
																				if top_k > self.model.config.num_labels:
																											a__:       Union[str, Any]											=self.model.config.num_labels
																				a__:       Union[str, Any]											=top_k
													return {}, {}, postprocess_params


						def   _lowerCamelCase			( self						:  Any      ,       _a						:  Union[str, Any]			):
													if isinstance(_a      ,       _a			):
																				if inputs.startswith("http://"			) or inputs.startswith("https://"			):
																											# We need to actually check for a real protocol, otherwise it's impossible to use a local file
																											# like http_huggingface_co.png
																											a__:       List[str]											=requests.get(_a			).content
																				else:
																											with open(_a      ,       "rb"			) as f:
																																		a__:       Dict											=f.read()

													if isinstance(_a      ,       _a			):
																				a__:       Tuple											=ffmpeg_read(_a      ,       self.feature_extractor.sampling_rate			)

													if isinstance(_a      ,       _a			):
																				# Accepting `"array"` which is the key defined in `datasets` for
																				# better integration
																				if not ("sampling_rate" in inputs and ("raw" in inputs or "array" in inputs)):
																											raise ValueError(
																											    "When passing a dictionary to AudioClassificationPipeline, the dict needs to contain a "
																											    "\"raw\" key containing the numpy array representing the audio and a \"sampling_rate\" key, "
																											    "containing the sampling_rate associated with that array"			)

																				a__:       Tuple											=inputs.pop("raw"      ,       _a			)
																				if _inputs is None:
																											# Remove path which will not be used from `datasets`.
																											inputs.pop("path"      ,       _a			)
																											a__:       int											=inputs.pop("array"      ,       _a			)
																				a__:       Optional[Any]											=inputs.pop("sampling_rate"			)
																				a__:       Optional[int]											=_inputs
																				if in_sampling_rate != self.feature_extractor.sampling_rate:
																											import torch

																											if is_torchaudio_available():
																																		from torchaudio import functional as F
																											else:
																																		raise ImportError(
																																		    "torchaudio is required to resample audio samples in AudioClassificationPipeline. "
																																		    "The torchaudio package can be installed through: `pip install torchaudio`."			)

																											a__:       Union[str, Any]											=F.resample(
																											    torch.from_numpy(_a			)      ,       _a      ,       self.feature_extractor.sampling_rate			).numpy()

													if not isinstance(_a      ,       np.ndarray			):
																				raise ValueError("We expect a numpy ndarray as input"			)
													if len(inputs.shape			) != 1:
																				raise ValueError("We expect a single channel audio input for AudioClassificationPipeline"			)

													a__:       Union[str, Any]											=self.feature_extractor(
													    _a      ,       sampling_rate=self.feature_extractor.sampling_rate      ,       return_tensors="pt"			)
													return processed


						def   _lowerCamelCase			( self						:  Dict      ,       _a						:  Tuple			):
													a__:       str											=self.model(**_a			)
													return model_outputs







						def   _lowerCamelCase			( self						:  Optional[Any]      ,       _a						:  Tuple      ,       _a						:  Tuple=5			):
													a__:       Tuple											=model_outputs.logits[0].softmax(-1			)
													a__							,			a__:       Any											=probs.topk(_a			)

													a__:       Union[str, Any]											=scores.tolist()
													a__:       Dict											=ids.tolist()

													a__:       Optional[int]											=[{"score": score, "label": self.model.config.idalabel[_id]} for score, _id in zip(_a      ,       _a			)]

													return labels





