




'''simple docstring'''



import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


UpperCAmelCase_       :  Any      =		logging.get_logger(__name__)

UpperCAmelCase_       :  List[str]      =		{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

UpperCAmelCase_       :  Union[str, Any]      =		{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

UpperCAmelCase_       :  List[str]      =		{
    'openai-gpt': 5_1_2,
}







def 			_UpperCamelCase (_lowerCamelCase							:     List[str]	)->   Union[str, Any]:




     '''simple docstring'''





     __snake_case    					=  text.strip()
     if not text:
          return []
     __snake_case    					=  text.split()
     return tokens





class     lowerCAmelCase   ( A__):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE=True    ,  __SCREAMING_SNAKE_CASE=None    ,  __SCREAMING_SNAKE_CASE=True    ,  __SCREAMING_SNAKE_CASE=None    ,  __SCREAMING_SNAKE_CASE=True    ,  )      ->  int:



            '''simple docstring'''




            if never_split is None:
                 __snake_case    					=  []
            __snake_case    					=  do_lower_case
            __snake_case    					=  set(_lowerCAmelCase  )
            __snake_case    					=  tokenize_chinese_chars
            __snake_case    					=  strip_accents
            __snake_case    					=  do_split_on_punc


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=None  )      ->  Dict:



            '''simple docstring'''




            __snake_case    					=  self.never_split.union(set(_lowerCAmelCase  )  ) if never_split else self.never_split
            __snake_case    					=  self._clean_text(_lowerCAmelCase  )

            # This was added on November 1st, 2018 for the multilingual and Chinese
            # models. This is also applied to the English models now, but it doesn't
            # matter since the English models were not trained on any Chinese data
            # and generally don't have any Chinese data in them (there are Chinese
            # characters in the vocabulary because Wikipedia does have some Chinese
            # words in the English Wikipedia.).
            if self.tokenize_chinese_chars:
                 __snake_case    					=  self._tokenize_chinese_chars(_lowerCAmelCase  )
            # prevents treating the same character with different unicode codepoints as different characters
            __snake_case    					=  unicodedata.normalize('''NFC'''    ,  _lowerCAmelCase  )
            __snake_case    					=  whitespace_tokenize(_lowerCAmelCase  )
            __snake_case    					=  []
            for token in orig_tokens:
                 if token not in never_split:
                      if self.do_lower_case:
                           __snake_case    					=  token.lower()
                           if self.strip_accents is not False:
                                __snake_case    					=  self._run_strip_accents(_lowerCAmelCase  )
                      elif self.strip_accents:
                           __snake_case    					=  self._run_strip_accents(_lowerCAmelCase  )
                 split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase    ,  _lowerCAmelCase  )  )

            __snake_case    					=  whitespace_tokenize(''' '''.join(_lowerCAmelCase  )  )
            return output_tokens


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[int]:



            '''simple docstring'''




            __snake_case    					=  unicodedata.normalize('''NFD'''    ,  _lowerCAmelCase  )
            __snake_case    					=  []
            for char in text:
                 __snake_case    					=  unicodedata.category(_lowerCAmelCase  )
                 if cat == "Mn":
                      continue
                 output.append(_lowerCAmelCase  )
            return "".join(_lowerCAmelCase  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=None  )      ->  List[Any]:



            '''simple docstring'''




            if not self.do_split_on_punc or (never_split is not None and text in never_split):
                 return [text]
            __snake_case    					=  list(_lowerCAmelCase  )
            __snake_case    					=  0
            __snake_case    					=  True
            __snake_case    					=  []
            while i < len(_lowerCAmelCase  ):
                 __snake_case    					=  chars[i]
                 if _is_punctuation(_lowerCAmelCase  ):
                      output.append([char]  )
                      __snake_case    					=  True
                 else:
                      if start_new_word:
                           output.append([]  )
                      __snake_case    					=  False
                      output[-1].append(_lowerCAmelCase  )
                 i += 1

            return ["".join(_lowerCAmelCase  ) for x in output]


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Union[str, Any]:



            '''simple docstring'''




            __snake_case    					=  []
            for char in text:
                 __snake_case    					=  ord(_lowerCAmelCase  )
                 if self._is_chinese_char(_lowerCAmelCase  ):
                      output.append(''' '''  )
                      output.append(_lowerCAmelCase  )
                      output.append(''' '''  )
                 else:
                      output.append(_lowerCAmelCase  )
            return "".join(_lowerCAmelCase  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  str:



            '''simple docstring'''




            if (
                (cp >= 0X4e00 and cp <= 0X9fff)
                or (cp >= 0X3400 and cp <= 0X4dbf)  #
                or (cp >= 0X20000 and cp <= 0X2a6df)  #
                or (cp >= 0X2a700 and cp <= 0X2b73f)  #
                or (cp >= 0X2b740 and cp <= 0X2b81f)  #
                or (cp >= 0X2b820 and cp <= 0X2ceaf)  #
                or (cp >= 0Xf900 and cp <= 0Xfaff)
                or (cp >= 0X2f800 and cp <= 0X2fa1f)  #
            ):  #
                 return True

            return False




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[int]:



            '''simple docstring'''




            __snake_case    					=  []
            for char in text:
                 __snake_case    					=  ord(_lowerCAmelCase  )
                 if cp == 0 or cp == 0Xfffd or _is_control(_lowerCAmelCase  ):
                      continue
                 if _is_whitespace(_lowerCAmelCase  ):
                      output.append(''' '''  )
                 else:
                      output.append(_lowerCAmelCase  )
            return "".join(_lowerCAmelCase  )







def 			_UpperCamelCase (_lowerCamelCase							:     Dict	)->   str:




     '''simple docstring'''





     __snake_case    					=  set()
     __snake_case    					=  word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)	)
          __snake_case    					=  char
     return pairs







def 			_UpperCamelCase (_lowerCamelCase							:     Dict	)->   List[str]:




     '''simple docstring'''





     __snake_case    					=  text.replace('''—'''   ,	'''-'''	)
     __snake_case    					=  text.replace('''–'''   ,	'''-'''	)
     __snake_case    					=  text.replace('''―'''   ,	'''-'''	)
     __snake_case    					=  text.replace('''…'''   ,	'''...'''	)
     __snake_case    					=  text.replace('''´'''   ,	'''\''''	)
     __snake_case    					=  re.sub(R'''(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''   ,	R''' \1 '''   ,	_lowerCamelCase	)
     __snake_case    					=  re.sub(R'''\s*\n\s*'''   ,	''' \n '''   ,	_lowerCamelCase	)
     __snake_case    					=  re.sub(R'''[^\S\n]+'''   ,	''' '''   ,	_lowerCamelCase	)
     return text.strip()







class     lowerCAmelCase   ( A__):
       __lowercase  :      int        =			VOCAB_FILES_NAMES
       __lowercase  :      str        =			PRETRAINED_VOCAB_FILES_MAP
       __lowercase  :      List[str]        =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __lowercase  :      Tuple        =			['''input_ids''', '''attention_mask''']


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE="<unk>"    ,  **__SCREAMING_SNAKE_CASE  )      ->  Union[str, Any]:



            '''simple docstring'''




            super().__init__(unk_token=_lowerCAmelCase    ,  **_lowerCAmelCase  )

            try:
                 import ftfy
                 from spacy.lang.en import English

                 __snake_case    					=  English()
                 __snake_case    					=  _nlp.tokenizer
                 __snake_case    					=  ftfy.fix_text
            except ImportError:
                 logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''  )
                 __snake_case    					=  BasicTokenizer(do_lower_case=_lowerCAmelCase  )
                 __snake_case    					=  None

            with open(_lowerCAmelCase    ,  encoding='''utf-8'''  ) as vocab_handle:
                 __snake_case    					=  json.load(_lowerCAmelCase  )
            __snake_case    					=  {v: k for k, v in self.encoder.items()}
            with open(_lowerCAmelCase    ,  encoding='''utf-8'''  ) as merges_handle:
                 __snake_case    					=  merges_handle.read().split('''\n'''  )[1:-1]
            __snake_case    					=  [tuple(merge.split()  ) for merge in merges]
            __snake_case    					=  dict(zip(_lowerCAmelCase    ,  range(len(_lowerCAmelCase  )  )  )  )
            __snake_case    					=  {}


       @property
       def       lowerCAmelCase      (   self  )      ->  Dict:



            '''simple docstring'''




            return True


       @property
       def       lowerCAmelCase      (   self  )      ->  Dict:



            '''simple docstring'''




            return len(self.encoder  )


       def       lowerCAmelCase      (   self  )      ->  int:



            '''simple docstring'''




            return dict(self.encoder    ,  **self.added_tokens_encoder  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Tuple:



            '''simple docstring'''




            __snake_case    					=  tuple(token[:-1]  ) + (token[-1] + '''</w>''',)
            if token in self.cache:
                 return self.cache[token]
            __snake_case    					=  get_pairs(_lowerCAmelCase  )

            if not pairs:
                 return token + "</w>"

            while True:
                 __snake_case    					=  min(_lowerCAmelCase    ,  key=lambda __SCREAMING_SNAKE_CASE  : self.bpe_ranks.get(_lowerCAmelCase    ,  float('''inf'''  )  )  )
                 if bigram not in self.bpe_ranks:
                      break
                 __snake_case					,    __snake_case    					=  bigram
                 __snake_case    					=  []
                 __snake_case    					=  0
                 while i < len(_lowerCAmelCase  ):
                      try:
                           __snake_case    					=  word.index(_lowerCAmelCase    ,  _lowerCAmelCase  )
                      except ValueError:
                           new_word.extend(word[i:]  )
                           break
                      else:
                           new_word.extend(word[i:j]  )
                           __snake_case    					=  j

                      if word[i] == first and i < len(_lowerCAmelCase  ) - 1 and word[i + 1] == second:
                           new_word.append(first + second  )
                           i += 2
                      else:
                           new_word.append(word[i]  )
                           i += 1
                 __snake_case    					=  tuple(_lowerCAmelCase  )
                 __snake_case    					=  new_word
                 if len(_lowerCAmelCase  ) == 1:
                      break
                 else:
                      __snake_case    					=  get_pairs(_lowerCAmelCase  )
            __snake_case    					=  ''' '''.join(_lowerCAmelCase  )
            if word == "\n  </w>":
                 __snake_case    					=  '''\n</w>'''
            __snake_case    					=  word
            return word


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  str:



            '''simple docstring'''




            __snake_case    					=  []
            if self.fix_text is None:
                 # Using BERT's BasicTokenizer
                 __snake_case    					=  self.nlp.tokenize(_lowerCAmelCase  )
                 for token in text:
                      split_tokens.extend(list(self.bpe(_lowerCAmelCase  ).split(''' '''  )  )  )
            else:
                 # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
                 __snake_case    					=  self.nlp(text_standardize(self.fix_text(_lowerCAmelCase  )  )  )
                 for token in text:
                      split_tokens.extend(list(self.bpe(token.text.lower()  ).split(''' '''  )  )  )
            return split_tokens


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Dict:



            '''simple docstring'''




            return self.encoder.get(_lowerCAmelCase    ,  self.encoder.get(self.unk_token  )  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Any:



            '''simple docstring'''




            return self.decoder.get(_lowerCAmelCase    ,  self.unk_token  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  List[Any]:



            '''simple docstring'''




            __snake_case    					=  ''''''.join(_lowerCAmelCase  ).replace('''</w>'''    ,  ''' '''  ).strip()
            return out_string




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE = None  )      ->  Optional[Any]:



            '''simple docstring'''




            if not os.path.isdir(_lowerCAmelCase  ):
                 logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
                 return
            __snake_case    					=  os.path.join(
                _lowerCAmelCase    ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
            __snake_case    					=  os.path.join(
                _lowerCAmelCase    ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']  )

            with open(_lowerCAmelCase    ,  '''w'''    ,  encoding='''utf-8'''  ) as f:
                 f.write(json.dumps(self.encoder    ,  indent=2    ,  sort_keys=_lowerCAmelCase    ,  ensure_ascii=_lowerCAmelCase  ) + '''\n'''  )

            __snake_case    					=  0
            with open(_lowerCAmelCase    ,  '''w'''    ,  encoding='''utf-8'''  ) as writer:
                 writer.write('''#version: 0.2\n'''  )
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,  key=lambda __SCREAMING_SNAKE_CASE  : kv[1]  ):
                      if index != token_index:
                           logger.warning(
                               F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                               ''' Please check that the tokenizer is not corrupted!'''  )
                           __snake_case    					=  token_index
                      writer.write(''' '''.join(_lowerCAmelCase  ) + '''\n'''  )
                      index += 1

            return vocab_file, merge_file












'''simple docstring'''



import warnings

from ...utils import logging
from .image_processing_maskformer import MaskFormerImageProcessor


UpperCAmelCase_       :  int      =		logging.get_logger(__name__)





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __init__(   self    ,  *__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )      ->  None:



            '''simple docstring'''




            warnings.warn(
                '''The class MaskFormerFeatureExtractor is deprecated and will be removed in version 5 of Transformers.'''
                ''' Please use MaskFormerImageProcessor instead.'''    ,  __SCREAMING_SNAKE_CASE    ,  )
            super().__init__(*__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )






