






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


UpperCAmelCase__			 =	logging.get_logger(__name__)

UpperCAmelCase__			 =	{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

UpperCAmelCase__			 =	{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

UpperCAmelCase__			 =	{
    'openai-gpt': 512,
}







def 						a_      (__A			)							->	Union[str, Any]:



     """simple docstring"""
     __a				:	Optional[Any]			=				text.strip()
     if not text:
          return []
     __a				:	Any			=				text.split()
     return tokens




class        snake_case_					(						A__			):

      """simple docstring"""



      def __init__(self:				Tuple   ,				__UpperCAmelCase:				Optional[int]=True   ,				__UpperCAmelCase:				Tuple=None   ,				__UpperCAmelCase:				str=True   ,				__UpperCAmelCase:				Tuple=None   ,				__UpperCAmelCase:				str=True   ,				)						->    List[Any]:




           '''simple docstring'''



           if never_split is None:
                __a				:	int			=				[]
           __a				:	int			=				do_lower_case
           __a				:	Optional[Any]			=				set(_lowerCAmelCase      )
           __a				:	Dict			=				tokenize_chinese_chars
           __a				:	Any			=				strip_accents
           __a				:	str			=				do_split_on_punc



      def 					UpperCAmelCase__				(self:				Tuple   ,				__UpperCAmelCase:				str   ,				__UpperCAmelCase:				List[Any]=None      )						->    Union[str, Any]:




           '''simple docstring'''



           __a				:	Tuple			=				self.never_split.union(set(_lowerCAmelCase      )      ) if never_split else self.never_split
           __a				:	Optional[int]			=				self._clean_text(_lowerCAmelCase      )

           # This was added on November 1st, 2018 for the multilingual and Chinese
           # models. This is also applied to the English models now, but it doesn't
           # matter since the English models were not trained on any Chinese data
           # and generally don't have any Chinese data in them (there are Chinese
           # characters in the vocabulary because Wikipedia does have some Chinese
           # words in the English Wikipedia.).
           if self.tokenize_chinese_chars:
                __a				:	List[str]			=				self._tokenize_chinese_chars(_lowerCAmelCase      )
           # prevents treating the same character with different unicode codepoints as different characters
           __a				:	Union[str, Any]			=				unicodedata.normalize("NFC"   ,				_lowerCAmelCase      )
           __a				:	Optional[int]			=				whitespace_tokenize(_lowerCAmelCase      )
           __a				:	Optional[int]			=				[]
           for token in orig_tokens:
                if token not in never_split:
                     if self.do_lower_case:
                          __a				:	Union[str, Any]			=				token.lower()
                          if self.strip_accents is not False:
                               __a				:	Optional[int]			=				self._run_strip_accents(_lowerCAmelCase      )
                     elif self.strip_accents:
                          __a				:	int			=				self._run_strip_accents(_lowerCAmelCase      )
                split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase   ,				_lowerCAmelCase      )      )

           __a				:	int			=				whitespace_tokenize(" ".join(_lowerCAmelCase      )      )
           return output_tokens



      def 					UpperCAmelCase__				(self:				List[Any]   ,				__UpperCAmelCase:				Union[str, Any]      )						->    str:




           '''simple docstring'''



           __a				:	Optional[Any]			=				unicodedata.normalize("NFD"   ,				_lowerCAmelCase      )
           __a				:	List[str]			=				[]
           for char in text:
                __a				:	str			=				unicodedata.category(_lowerCAmelCase      )
                if cat == "Mn":
                     continue
                output.append(_lowerCAmelCase      )
           return "".join(_lowerCAmelCase      )



      def 					UpperCAmelCase__				(self:				Dict   ,				__UpperCAmelCase:				Dict   ,				__UpperCAmelCase:				Any=None      )						->    int:




           '''simple docstring'''



           if not self.do_split_on_punc or (never_split is not None and text in never_split):
                return [text]
           __a				:	Tuple			=				list(_lowerCAmelCase      )
           __a				:	Dict			=				0
           __a				:	Optional[Any]			=				True
           __a				:	List[Any]			=				[]
           while i < len(_lowerCAmelCase      ):
                __a				:	int			=				chars[i]
                if _is_punctuation(_lowerCAmelCase      ):
                     output.append([char]      )
                     __a				:	str			=				True
                else:
                     if start_new_word:
                          output.append([]      )
                     __a				:	List[Any]			=				False
                     output[-1].append(_lowerCAmelCase      )
                i += 1

           return ["".join(_lowerCAmelCase      ) for x in output]



      def 					UpperCAmelCase__				(self:				List[Any]   ,				__UpperCAmelCase:				Dict      )						->    List[str]:




           '''simple docstring'''



           __a				:	Any			=				[]
           for char in text:
                __a				:	int			=				ord(_lowerCAmelCase      )
                if self._is_chinese_char(_lowerCAmelCase      ):
                     output.append(" "      )
                     output.append(_lowerCAmelCase      )
                     output.append(" "      )
                else:
                     output.append(_lowerCAmelCase      )
           return "".join(_lowerCAmelCase      )



      def 					UpperCAmelCase__				(self:				Optional[int]   ,				__UpperCAmelCase:				Any      )						->    int:




           '''simple docstring'''



           if (
               (cp >= 0X4E00 and cp <= 0X9FFF)
               or (cp >= 0X3400 and cp <= 0X4DBF)  #
               or (cp >= 0X2_0000 and cp <= 0X2_A6DF)  #
               or (cp >= 0X2_A700 and cp <= 0X2_B73F)  #
               or (cp >= 0X2_B740 and cp <= 0X2_B81F)  #
               or (cp >= 0X2_B820 and cp <= 0X2_CEAF)  #
               or (cp >= 0XF900 and cp <= 0XFAFF)
               or (cp >= 0X2_F800 and cp <= 0X2_FA1F)  #
           ):  #
                return True

           return False



      def 					UpperCAmelCase__				(self:				Dict   ,				__UpperCAmelCase:				Optional[Any]      )						->    List[Any]:




           '''simple docstring'''



           __a				:	Optional[Any]			=				[]
           for char in text:
                __a				:	List[str]			=				ord(_lowerCAmelCase      )
                if cp == 0 or cp == 0XFFFD or _is_control(_lowerCAmelCase      ):
                     continue
                if _is_whitespace(_lowerCAmelCase      ):
                     output.append(" "      )
                else:
                     output.append(_lowerCAmelCase      )
           return "".join(_lowerCAmelCase      )







def 						a_      (__A			)							->	str:



     """simple docstring"""
     __a				:	List[str]			=				set()
     __a				:	int			=				word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)			)
          __a				:	Any			=				char
     return pairs







def 						a_      (__A			)							->	List[str]:



     """simple docstring"""
     __a				:	str			=				text.replace("—"							,					"-"			)
     __a				:	int			=				text.replace("–"							,					"-"			)
     __a				:	Dict			=				text.replace("―"							,					"-"			)
     __a				:	Union[str, Any]			=				text.replace("…"							,					"..."			)
     __a				:	str			=				text.replace("´"							,					"'"			)
     __a				:	List[str]			=				re.sub(r"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"							,					r" \1 "							,					__A			)
     __a				:	Optional[Any]			=				re.sub(r"\s*\n\s*"							,					" \n "							,					__A			)
     __a				:	Optional[int]			=				re.sub(r"[^\S\n]+"							,					" "							,					__A			)
     return text.strip()




class        snake_case_					(						A__			):

      """simple docstring"""



      snake_case__   					=			VOCAB_FILES_NAMES
      snake_case__   					=			PRETRAINED_VOCAB_FILES_MAP
      snake_case__   					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      snake_case__   					=			["""input_ids""", """attention_mask"""]



      def __init__(self:				Optional[Any]   ,				__UpperCAmelCase:				Any   ,				__UpperCAmelCase:				Tuple   ,				__UpperCAmelCase:				Tuple="<unk>"   ,				**__UpperCAmelCase:				str      )						->    Any:




           '''simple docstring'''



           super().__init__(unk_token=_lowerCAmelCase   ,				**_lowerCAmelCase      )

           try:
                import ftfy
                from spacy.lang.en import English

                __a				:	Union[str, Any]			=				English()
                __a				:	Dict			=				_nlp.tokenizer
                __a				:	List[str]			=				ftfy.fix_text
           except ImportError:
                logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."      )
                __a				:	List[str]			=				BasicTokenizer(do_lower_case=_lowerCAmelCase      )
                __a				:	Optional[Any]			=				None

           with open(_lowerCAmelCase   ,				encoding="utf-8"      ) as vocab_handle:
                __a				:	Tuple			=				json.load(_lowerCAmelCase      )
           __a				:	List[Any]			=				{v: k for k, v in self.encoder.items()}
           with open(_lowerCAmelCase   ,				encoding="utf-8"      ) as merges_handle:
                __a				:	Tuple			=				merges_handle.read().split("\n"      )[1:-1]
           __a				:	List[str]			=				[tuple(merge.split()      ) for merge in merges]
           __a				:	List[str]			=				dict(zip(_lowerCAmelCase   ,				range(len(_lowerCAmelCase      )      )      )      )
           __a				:	Any			=				{}



      @property
      def 					UpperCAmelCase__				(self:				Any      )						->    List[str]:




           '''simple docstring'''



           return True



      @property
      def 					UpperCAmelCase__				(self:				List[Any]      )						->    Optional[Any]:




           '''simple docstring'''



           return len(self.encoder      )



      def 					UpperCAmelCase__				(self:				List[Any]      )						->    Any:




           '''simple docstring'''



           return dict(self.encoder   ,				**self.added_tokens_encoder      )



      def 					UpperCAmelCase__				(self:				Optional[int]   ,				__UpperCAmelCase:				Tuple      )						->    List[str]:




           '''simple docstring'''



           __a				:	Dict			=				tuple(token[:-1]      ) + (token[-1] + "</w>",)
           if token in self.cache:
                return self.cache[token]
           __a				:	str			=				get_pairs(_lowerCAmelCase      )

           if not pairs:
                return token + "</w>"

           while True:
                __a				:	int			=				min(_lowerCAmelCase   ,				key=lambda __UpperCAmelCase      : self.bpe_ranks.get(_lowerCAmelCase   ,				float("inf"      )      )      )
                if bigram not in self.bpe_ranks:
                     break
                __a			, __a				:	str			=				bigram
                __a				:	str			=				[]
                __a				:	List[Any]			=				0
                while i < len(_lowerCAmelCase      ):
                     try:
                          __a				:	Optional[int]			=				word.index(_lowerCAmelCase   ,				_lowerCAmelCase      )
                     except ValueError:
                          new_word.extend(word[i:]      )
                          break
                     else:
                          new_word.extend(word[i:j]      )
                          __a				:	str			=				j

                     if word[i] == first and i < len(_lowerCAmelCase      ) - 1 and word[i + 1] == second:
                          new_word.append(first + second      )
                          i += 2
                     else:
                          new_word.append(word[i]      )
                          i += 1
                __a				:	Any			=				tuple(_lowerCAmelCase      )
                __a				:	Optional[int]			=				new_word
                if len(_lowerCAmelCase      ) == 1:
                     break
                else:
                     __a				:	Union[str, Any]			=				get_pairs(_lowerCAmelCase      )
           __a				:	List[str]			=				" ".join(_lowerCAmelCase      )
           if word == "\n  </w>":
                __a				:	str			=				"\n</w>"
           __a				:	Any			=				word
           return word



      def 					UpperCAmelCase__				(self:				str   ,				__UpperCAmelCase:				List[Any]      )						->    List[str]:




           '''simple docstring'''



           __a				:	Optional[int]			=				[]
           if self.fix_text is None:
                # Using BERT's BasicTokenizer
                __a				:	Any			=				self.nlp.tokenize(_lowerCAmelCase      )
                for token in text:
                     split_tokens.extend(list(self.bpe(_lowerCAmelCase      ).split(" "      )      )      )
           else:
                # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
                __a				:	Tuple			=				self.nlp(text_standardize(self.fix_text(_lowerCAmelCase      )      )      )
                for token in text:
                     split_tokens.extend(list(self.bpe(token.text.lower()      ).split(" "      )      )      )
           return split_tokens



      def 					UpperCAmelCase__				(self:				str   ,				__UpperCAmelCase:				Optional[int]      )						->    Optional[Any]:




           '''simple docstring'''



           return self.encoder.get(_lowerCAmelCase   ,				self.encoder.get(self.unk_token      )      )



      def 					UpperCAmelCase__				(self:				Optional[int]   ,				__UpperCAmelCase:				str      )						->    Optional[Any]:




           '''simple docstring'''



           return self.decoder.get(_lowerCAmelCase   ,				self.unk_token      )



      def 					UpperCAmelCase__				(self:				List[Any]   ,				__UpperCAmelCase:				List[str]      )						->    Tuple:




           '''simple docstring'''



           __a				:	List[str]			=				"".join(_lowerCAmelCase      ).replace("</w>"   ,				" "      ).strip()
           return out_string



      def 					UpperCAmelCase__				(self:				str   ,				__UpperCAmelCase:				str   ,				__UpperCAmelCase:				Optional[str] = None      )						->    List[Any]:




           '''simple docstring'''



           if not os.path.isdir(_lowerCAmelCase      ):
                logger.error(f'Vocabulary path ({save_directory}) should be a directory'      )
                return
           __a				:	Dict			=				os.path.join(
               _lowerCAmelCase   ,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )
           __a				:	List[Any]			=				os.path.join(
               _lowerCAmelCase   ,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]      )

           with open(_lowerCAmelCase   ,				"w"   ,				encoding="utf-8"      ) as f:
                f.write(json.dumps(self.encoder   ,				indent=2   ,				sort_keys=_lowerCAmelCase   ,				ensure_ascii=_lowerCAmelCase      ) + "\n"      )

           __a				:	Dict			=				0
           with open(_lowerCAmelCase   ,				"w"   ,				encoding="utf-8"      ) as writer:
                writer.write("#version: 0.2\n"      )
                for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,				key=lambda __UpperCAmelCase      : kv[1]      ):
                     if index != token_index:
                          logger.warning(
                              f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                              " Please check that the tokenizer is not corrupted!"      )
                          __a				:	List[str]			=				token_index
                     writer.write(" ".join(_lowerCAmelCase      ) + "\n"      )
                     index += 1

           return vocab_file, merge_file













import warnings

from .generation import GenerationMixin




class        snake_case_					(						__UpperCamelCase			):

						"""simple docstring"""



						warnings.warn(
						    """Importing `GenerationMixin` from `src/transformers/generation_utils.py` is deprecated and will """
						    """be removed in Transformers v5. Import as `from transformers import GenerationMixin` instead."""		,       __UpperCamelCase		,       )





