



'''simple docstring'''

import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase		    =     logging.get_logger(__name__)
__lowerCAmelCase		    =     {'''vocab_file''': '''spiece.model'''}

__lowerCAmelCase		    =     {
    '''vocab_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
    },
}

__lowerCAmelCase		    =     {
    '''google/fnet-base''': 512,
    '''google/fnet-large''': 512,
}

__lowerCAmelCase		    =     '''▁'''

class 			__magic_name__		(					_UpperCamelCase     ):
							lowerCAmelCase :					Dict            =      VOCAB_FILES_NAMES
							lowerCAmelCase :					List[Any]            =      PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase :					Dict            =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase :					List[Any]            =      ['input_ids', 'token_type_ids']







							def __init__(							self      : Union[str, Any]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : List[Any]=False			,_UpperCAmelCase      : Any=True			,_UpperCAmelCase      : Optional[int]=True			,_UpperCAmelCase      : Optional[Any]="<unk>"			,_UpperCAmelCase      : int="[SEP]"			,_UpperCAmelCase      : int="<pad>"			,_UpperCAmelCase      : Any="[CLS]"			,_UpperCAmelCase      : Optional[int]="[MASK]"			,_UpperCAmelCase      : Optional[Dict[str, Any]] = None			,**_UpperCAmelCase      : List[str]			,):
									# Mask token behave like a normal word, i.e. include the space before it and
									# is included in the raw text, there should be a match in a non-normalized sentence.
									_a  :   Tuple				 =							(
									    AddedToken(_UpperCAmelCase			,lstrip=_UpperCAmelCase			,rstrip=_UpperCAmelCase			,normalized=_UpperCAmelCase	)
									    if isinstance(_UpperCAmelCase			,_UpperCAmelCase	)
									    else mask_token
									)

									_a  :   Any				 =							{} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    do_lower_case=_UpperCAmelCase			,remove_space=_UpperCAmelCase			,keep_accents=_UpperCAmelCase			,unk_token=_UpperCAmelCase			,sep_token=_UpperCAmelCase			,pad_token=_UpperCAmelCase			,cls_token=_UpperCAmelCase			,mask_token=_UpperCAmelCase			,sp_model_kwargs=self.sp_model_kwargs			,**_UpperCAmelCase			,)

									_a  :   Union[str, Any]				 =							do_lower_case
									_a  :   Dict				 =							remove_space
									_a  :   str				 =							keep_accents
									_a  :   Dict				 =							vocab_file

									_a  :   Union[str, Any]				 =							spm.SentencePieceProcessor(**self.sp_model_kwargs	)
									self.sp_model.Load(_UpperCAmelCase	)







							@property
							def 							__lowercase    (							self      : List[str]	):
									return len(self.sp_model	)







							def 							__lowercase    (							self      : Union[str, Any]	):
									_a  :   Tuple				 =							{self.convert_ids_to_tokens(_UpperCAmelCase	): i for i in range(self.vocab_size	)}
									vocab.update(self.added_tokens_encoder	)
									return vocab







							def __getstate__(							self      : Optional[int]	):
									_a  :   List[str]				 =							self.__dict__.copy()
									_a  :   Optional[int]				 =							None
									return state







							def __setstate__(							self      : str			,_UpperCAmelCase      : Optional[int]	):
									_a  :   Tuple				 =							d

									# for backward compatibility
									if not hasattr(self			,'sp_model_kwargs'	):
											_a  :   Any				 =							{}

									_a  :   List[str]				 =							spm.SentencePieceProcessor(**self.sp_model_kwargs	)
									self.sp_model.Load(self.vocab_file	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : List[str]	):
									if self.remove_space:
											_a  :   List[Any]				 =							' '.join(inputs.strip().split()	)
									else:
											_a  :   Tuple				 =							inputs
									_a  :   Tuple				 =							outputs.replace('``'			,'"'	).replace('\'\''			,'"'	)

									if not self.keep_accents:
											_a  :   Tuple				 =							unicodedata.normalize('NFKD'			,_UpperCAmelCase	)
											_a  :   int				 =							''.join([c for c in outputs if not unicodedata.combining(_UpperCAmelCase	)]	)
									if self.do_lower_case:
											_a  :   Any				 =							outputs.lower()

									return outputs







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : str	):
									_a  :   Tuple				 =							self.preprocess_text(_UpperCAmelCase	)
									_a  :   Any				 =							self.sp_model.encode(_UpperCAmelCase			,out_type=_UpperCAmelCase	)
									_a  :   Tuple				 =							[]
									for piece in pieces:
											if len(_UpperCAmelCase	) > 1 and piece[-1] == str(','	) and piece[-2].isdigit():
													_a  :   List[str]				 =							self.sp_model.EncodeAsPieces(piece[:-1].replace(_UpperCAmelCase			,''	)	)
													if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
															if len(cur_pieces[0]	) == 1:
																	_a  :   List[Any]				 =							cur_pieces[1:]
															else:
																	_a  :   Union[str, Any]				 =							cur_pieces[0][1:]
													cur_pieces.append(piece[-1]	)
													new_pieces.extend(_UpperCAmelCase	)
											else:
													new_pieces.append(_UpperCAmelCase	)

									return new_pieces







							def 							__lowercase    (							self      : int			,_UpperCAmelCase      : Any	):
									return self.sp_model.PieceToId(_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any			,_UpperCAmelCase      : int	):
									return self.sp_model.IdToPiece(_UpperCAmelCase	)







							def 							__lowercase    (							self      : str			,_UpperCAmelCase      : Any	):
									_a  :   Any				 =							[]
									_a  :   Optional[Any]				 =							''
									_a  :   Optional[Any]				 =							False
									for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
													if not prev_is_special:
															out_string += " "
													out_string += self.sp_model.decode(_UpperCAmelCase	) + token
													_a  :   Dict				 =							True
													_a  :   Union[str, Any]				 =							[]
											else:
													current_sub_tokens.append(_UpperCAmelCase	)
													_a  :   int				 =							False
									out_string += self.sp_model.decode(_UpperCAmelCase	)
									return out_string.strip()







							def 							__lowercase    (							self      : List[str]			,_UpperCAmelCase      : List[int]			,_UpperCAmelCase      : bool = False			,_UpperCAmelCase      : bool = None			,_UpperCAmelCase      : bool = True			,**_UpperCAmelCase      : Optional[int]			,):
									_a  :   Union[str, Any]				 =							kwargs.pop('use_source_tokenizer'			,_UpperCAmelCase	)

									_a  :   Dict				 =							self.convert_ids_to_tokens(_UpperCAmelCase			,skip_special_tokens=_UpperCAmelCase	)

									# To avoid mixing byte-level and unicode for byte-level BPT
									# we need to build string separately for added tokens and byte-level tokens
									# cf. https://github.com/huggingface/transformers/issues/1133
									_a  :   Optional[int]				 =							[]
									_a  :   Optional[int]				 =							[]
									for token in filtered_tokens:
											if skip_special_tokens and token in self.all_special_ids:
													continue
											if token in self.added_tokens_encoder:
													if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(_UpperCAmelCase	)	)
															_a  :   Optional[int]				 =							[]
													sub_texts.append(_UpperCAmelCase	)
											else:
													current_sub_text.append(_UpperCAmelCase	)
									if current_sub_text:
											sub_texts.append(self.convert_tokens_to_string(_UpperCAmelCase	)	)

									# Mimic the behavior of the Rust tokenizer:
									# No space after <unk>
									if spaces_between_special_tokens:
											_a  :   Tuple				 =							re.sub(R'(<unk>) '			,R'\1'			,' '.join(_UpperCAmelCase	)	)
									else:
											_a  :   Any				 =							''.join(_UpperCAmelCase	)

									_a  :   int				 =							(
									    clean_up_tokenization_spaces
									    if clean_up_tokenization_spaces is not None
									    else self.clean_up_tokenization_spaces
									)
									if clean_up_tokenization_spaces:
											_a  :   Any				 =							self.clean_up_tokenization(_UpperCAmelCase	)
											return clean_text
									else:
											return text







							def 							__lowercase    (							self      : Any			,_UpperCAmelCase      : List[int]			,_UpperCAmelCase      : Optional[List[int]] = None	):
									_a  :   List[Any]				 =							[self.sep_token_id]
									_a  :   Optional[Any]				 =							[self.cls_token_id]
									if token_ids_a is None:
											return cls + token_ids_a + sep
									return cls + token_ids_a + sep + token_ids_a + sep







							def 							__lowercase    (							self      : int			,_UpperCAmelCase      : List[int]			,_UpperCAmelCase      : Optional[List[int]] = None			,_UpperCAmelCase      : bool = False	):

									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=_UpperCAmelCase			,token_ids_a=_UpperCAmelCase			,already_has_special_tokens=_UpperCAmelCase	)

									if token_ids_a is not None:
											return [1] + ([0] * len(_UpperCAmelCase	)) + [1] + ([0] * len(_UpperCAmelCase	)) + [1]
									return [1] + ([0] * len(_UpperCAmelCase	)) + [1]







							def 							__lowercase    (							self      : Dict			,_UpperCAmelCase      : List[int]			,_UpperCAmelCase      : Optional[List[int]] = None	):
									_a  :   List[str]				 =							[self.sep_token_id]
									_a  :   Tuple				 =							[self.cls_token_id]

									if token_ids_a is None:
											return len(cls + token_ids_a + sep	) * [0]
									return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







							def 							__lowercase    (							self      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Optional[str] = None	):
									if not os.path.isdir(_UpperCAmelCase	):
											logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""	)
											return
									_a  :   Dict				 =							os.path.join(
									    _UpperCAmelCase			,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

									if os.path.abspath(self.vocab_file	) != os.path.abspath(_UpperCAmelCase	) and os.path.isfile(self.vocab_file	):
											copyfile(self.vocab_file			,_UpperCAmelCase	)
									elif not os.path.isfile(self.vocab_file	):
											with open(_UpperCAmelCase			,'wb'	) as fi:
													_a  :   Optional[Any]				 =							self.sp_model.serialized_model_proto()
													fi.write(_UpperCAmelCase	)

									return (out_vocab_file,)









'''simple docstring'''

from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    get_resize_output_image_size,
    normalize,
    pad,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging
from ...utils.import_utils import is_vision_available


__lowerCAmelCase		    =     logging.get_logger(__name__)


if is_vision_available():
							import PIL

class 			__magic_name__		(					_UpperCamelCase     ):
							lowerCAmelCase :					List[str]            =      ['pixel_values']







							def __init__(							self      : int			,_UpperCAmelCase      : bool = True			,_UpperCAmelCase      : Dict[str, int] = None			,_UpperCAmelCase      : PILImageResampling = PILImageResampling.BILINEAR			,_UpperCAmelCase      : bool = True			,_UpperCAmelCase      : bool = False			,_UpperCAmelCase      : bool = True			,_UpperCAmelCase      : bool = True			,_UpperCAmelCase      : Union[int, float] = 1 / 255			,_UpperCAmelCase      : bool = True			,_UpperCAmelCase      : Optional[Union[float, List[float]]] = None			,_UpperCAmelCase      : Optional[Union[float, List[float]]] = None			,**_UpperCAmelCase      : Optional[Any]			,):
									super().__init__(**_UpperCAmelCase	)

									_a  :   Union[str, Any]				 =							size if size is not None else {'height': 2560, 'width': 1920}
									if isinstance(_UpperCAmelCase			,(tuple, list)	):
											# The previous feature extractor size parameter was in (width, height) format
											_a  :   Union[str, Any]				 =							size[::-1]
									_a  :   Dict				 =							get_size_dict(_UpperCAmelCase	)

									_a  :   int				 =							do_resize
									_a  :   Any				 =							size
									_a  :   Any				 =							resample
									_a  :   int				 =							do_thumbnail
									_a  :   Tuple				 =							do_align_long_axis
									_a  :   Optional[Any]				 =							do_pad
									_a  :   Any				 =							do_rescale
									_a  :   List[str]				 =							rescale_factor
									_a  :   Any				 =							do_normalize
									_a  :   Union[str, Any]				 =							image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
									_a  :   Any				 =							image_std if image_std is not None else IMAGENET_STANDARD_STD







							def 							__lowercase    (							self      : Optional[Any]			,_UpperCAmelCase      : np.ndarray			,_UpperCAmelCase      : Dict[str, int]			,_UpperCAmelCase      : Optional[Union[str, ChannelDimension]] = None	):
									_a ,      _a  :   List[Any]				 =							get_image_size(_UpperCAmelCase	)
									_a ,      _a  :   List[str]				 =							size['height'], size['width']

									if (output_width < output_height and input_width > input_height) or (
									    output_width > output_height and input_width < input_height
									):
											_a  :   Optional[int]				 =							np.rotaa(_UpperCAmelCase			,3	)

									if data_format is not None:
											_a  :   Any				 =							to_channel_dimension_format(_UpperCAmelCase			,_UpperCAmelCase	)

									return image







							def 							__lowercase    (							self      : Any			,_UpperCAmelCase      : np.ndarray			,_UpperCAmelCase      : Dict[str, int]			,_UpperCAmelCase      : bool = False			,_UpperCAmelCase      : Optional[Union[str, ChannelDimension]] = None			,):
									_a ,      _a  :   List[Any]				 =							size['height'], size['width']
									_a ,      _a  :   str				 =							get_image_size(_UpperCAmelCase	)

									_a  :   List[str]				 =							output_width - input_width
									_a  :   Union[str, Any]				 =							output_height - input_height

									if random_padding:
											_a  :   Union[str, Any]				 =							np.random.randint(low=0			,high=delta_height + 1	)
											_a  :   Any				 =							np.random.randint(low=0			,high=delta_width + 1	)
									else:
											_a  :   Tuple				 =							delta_height // 2
											_a  :   List[Any]				 =							delta_width // 2

									_a  :   Dict				 =							delta_height - pad_top
									_a  :   Optional[int]				 =							delta_width - pad_left

									_a  :   int				 =							((pad_top, pad_bottom), (pad_left, pad_right))
									return pad(_UpperCAmelCase			,_UpperCAmelCase			,data_format=_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[Any]			,*_UpperCAmelCase      : int			,**_UpperCAmelCase      : List[str]	):
									logger.info('pad is deprecated and will be removed in version 4.27. Please use pad_image instead.'	)
									return self.pad_image(*_UpperCAmelCase			,**_UpperCAmelCase	)







							def 							__lowercase    (							self      : Dict			,_UpperCAmelCase      : np.ndarray			,_UpperCAmelCase      : Dict[str, int]			,_UpperCAmelCase      : PILImageResampling = PILImageResampling.BICUBIC			,_UpperCAmelCase      : Optional[Union[str, ChannelDimension]] = None			,**_UpperCAmelCase      : Union[str, Any]			,):
									_a ,      _a  :   Any				 =							get_image_size(_UpperCAmelCase	)
									_a ,      _a  :   Union[str, Any]				 =							size['height'], size['width']

									# We always resize to the smallest of either the input or output size.
									_a  :   Dict				 =							min(_UpperCAmelCase			,_UpperCAmelCase	)
									_a  :   List[Any]				 =							min(_UpperCAmelCase			,_UpperCAmelCase	)

									if height == input_height and width == input_width:
											return image

									if input_height > input_width:
											_a  :   Any				 =							int(input_width * height / input_height	)
									elif input_width > input_height:
											_a  :   int				 =							int(input_height * width / input_width	)

									return resize(
									    _UpperCAmelCase			,size=(height, width)			,resample=_UpperCAmelCase			,reducing_gap=2.0			,data_format=_UpperCAmelCase			,**_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[str]			,_UpperCAmelCase      : np.ndarray			,_UpperCAmelCase      : Dict[str, int]			,_UpperCAmelCase      : PILImageResampling = PILImageResampling.BICUBIC			,_UpperCAmelCase      : Optional[Union[str, ChannelDimension]] = None			,**_UpperCAmelCase      : List[Any]			,):
									_a  :   List[Any]				 =							get_size_dict(_UpperCAmelCase	)
									_a  :   int				 =							min(size['height']			,size['width']	)
									_a  :   Dict				 =							get_resize_output_image_size(_UpperCAmelCase			,size=_UpperCAmelCase			,default_to_square=_UpperCAmelCase	)
									_a  :   int				 =							resize(_UpperCAmelCase			,size=_UpperCAmelCase			,resample=_UpperCAmelCase			,data_format=_UpperCAmelCase			,**_UpperCAmelCase	)
									return resized_image







							def 							__lowercase    (							self      : int			,_UpperCAmelCase      : np.ndarray			,_UpperCAmelCase      : Union[int, float]			,_UpperCAmelCase      : Optional[Union[str, ChannelDimension]] = None			,**_UpperCAmelCase      : Optional[Any]			,):
									return rescale(_UpperCAmelCase			,scale=_UpperCAmelCase			,data_format=_UpperCAmelCase			,**_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[str]			,_UpperCAmelCase      : np.ndarray			,_UpperCAmelCase      : Union[float, List[float]]			,_UpperCAmelCase      : Union[float, List[float]]			,_UpperCAmelCase      : Optional[Union[str, ChannelDimension]] = None			,**_UpperCAmelCase      : Dict			,):
									return normalize(_UpperCAmelCase			,mean=_UpperCAmelCase			,std=_UpperCAmelCase			,data_format=_UpperCAmelCase			,**_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : ImageInput			,_UpperCAmelCase      : bool = None			,_UpperCAmelCase      : Dict[str, int] = None			,_UpperCAmelCase      : PILImageResampling = None			,_UpperCAmelCase      : bool = None			,_UpperCAmelCase      : bool = None			,_UpperCAmelCase      : bool = None			,_UpperCAmelCase      : bool = False			,_UpperCAmelCase      : bool = None			,_UpperCAmelCase      : float = None			,_UpperCAmelCase      : bool = None			,_UpperCAmelCase      : Optional[Union[float, List[float]]] = None			,_UpperCAmelCase      : Optional[Union[float, List[float]]] = None			,_UpperCAmelCase      : Optional[Union[str, TensorType]] = None			,_UpperCAmelCase      : Optional[ChannelDimension] = ChannelDimension.FIRST			,**_UpperCAmelCase      : int			,):
									_a  :   List[str]				 =							do_resize if do_resize is not None else self.do_resize
									_a  :   Optional[Any]				 =							size if size is not None else self.size
									if isinstance(_UpperCAmelCase			,(tuple, list)	):
											# Previous feature extractor had size in (width, height) format
											_a  :   Optional[int]				 =							size[::-1]
									_a  :   Tuple				 =							get_size_dict(_UpperCAmelCase	)
									_a  :   Optional[int]				 =							resample if resample is not None else self.resample
									_a  :   Union[str, Any]				 =							do_thumbnail if do_thumbnail is not None else self.do_thumbnail
									_a  :   int				 =							do_align_long_axis if do_align_long_axis is not None else self.do_align_long_axis
									_a  :   Optional[Any]				 =							do_pad if do_pad is not None else self.do_pad
									_a  :   str				 =							do_rescale if do_rescale is not None else self.do_rescale
									_a  :   Dict				 =							rescale_factor if rescale_factor is not None else self.rescale_factor
									_a  :   Dict				 =							do_normalize if do_normalize is not None else self.do_normalize
									_a  :   Optional[int]				 =							image_mean if image_mean is not None else self.image_mean
									_a  :   Dict				 =							image_std if image_std is not None else self.image_std

									_a  :   Optional[Any]				 =							make_list_of_images(_UpperCAmelCase	)

									if not valid_images(_UpperCAmelCase	):
											raise ValueError(
											    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
											    'torch.Tensor, tf.Tensor or jax.ndarray.'	)

									if do_resize and size is None:
											raise ValueError('Size must be specified if do_resize is True.'	)

									if do_rescale and rescale_factor is None:
											raise ValueError('Rescale factor must be specified if do_rescale is True.'	)

									if do_pad and size is None:
											raise ValueError('Size must be specified if do_pad is True.'	)

									if do_normalize and (image_mean is None or image_std is None):
											raise ValueError('Image mean and std must be specified if do_normalize is True.'	)

									# All transformations expect numpy arrays.
									_a  :   Dict				 =							[to_numpy_array(_UpperCAmelCase	) for image in images]

									if do_align_long_axis:
											_a  :   Optional[Any]				 =							[self.align_long_axis(_UpperCAmelCase			,size=_UpperCAmelCase	) for image in images]

									if do_resize:
											_a  :   List[Any]				 =							[self.resize(image=_UpperCAmelCase			,size=_UpperCAmelCase			,resample=_UpperCAmelCase	) for image in images]

									if do_thumbnail:
											_a  :   Dict				 =							[self.thumbnail(image=_UpperCAmelCase			,size=_UpperCAmelCase	) for image in images]

									if do_pad:
											_a  :   List[str]				 =							[self.pad_image(image=_UpperCAmelCase			,size=_UpperCAmelCase			,random_padding=_UpperCAmelCase	) for image in images]

									if do_rescale:
											_a  :   List[str]				 =							[self.rescale(image=_UpperCAmelCase			,scale=_UpperCAmelCase	) for image in images]

									if do_normalize:
											_a  :   List[str]				 =							[self.normalize(image=_UpperCAmelCase			,mean=_UpperCAmelCase			,std=_UpperCAmelCase	) for image in images]

									_a  :   Optional[Any]				 =							[to_channel_dimension_format(_UpperCAmelCase			,_UpperCAmelCase	) for image in images]

									_a  :   List[Any]				 =							{'pixel_values': images}
									return BatchFeature(data=_UpperCAmelCase			,tensor_type=_UpperCAmelCase	)




