
"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation

lowercase_						    =					logging.get_logger(__name__)

lowercase_						    =					{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

lowercase_						    =					{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
}

lowercase_						    =					{
    "gpt2": 1024,
    "gpt2-medium": 1024,
    "gpt2-large": 1024,
    "gpt2-xl": 1024,
    "distilgpt2": 1024,
}




@lru_cache()
def 			A_			(					)     ->							Any:


     """simple docstring"""







     UpperCAmelCase_						:    Any    					=  (
         list(range(ord("""!"""      )						, ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      )						, ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      )						, ord("""ÿ"""      ) + 1      )      )
     )
     UpperCAmelCase_						:    List[Any]    					=  bs[:]
     UpperCAmelCase_						:    Optional[Any]    					=  0
     for b in range(2**8      ):
          if b not in bs:
               bs.append(lowercase      )
               cs.append(2**8 + n      )
               n += 1
     UpperCAmelCase_						:    Tuple    					=  [chr(lowercase      ) for n in cs]
     return dict(zip(lowercase						, lowercase      )      )




def 			A_			(					lowercase      )     ->							Dict:


     """simple docstring"""







     UpperCAmelCase_						:    Optional[int]    					=  set()
     UpperCAmelCase_						:    int    					=  word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)      )
          UpperCAmelCase_						:    List[Any]    					=  char
     return pairs



class 		UpperCAmelCase_	(lowerCamelCase_						):
 """simple docstring"""




 UpperCamelCase_						:     List[str]				   =    VOCAB_FILES_NAMES
 UpperCamelCase_						:     Any				   =    PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase_						:     Tuple				   =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase_						:     Union[str, Any]				   =    ["""input_ids""", """attention_mask"""]




 def __init__( self       : Dict     ,   a_       : List[Any]     ,   a_       : List[str]     ,   a_       : Any="replace"     ,   a_       : List[Any]="<|endoftext|>"     ,   a_       : str="<|endoftext|>"     ,   a_       : Dict="<|endoftext|>"     ,   a_       : Dict=None     ,   a_       : str=False     ,   a_       : Optional[Any]=False     ,   **a_       : Union[str, Any]     ,   )->					Dict:



      """simple docstring"""

      UpperCAmelCase_						:    List[str]    					=  AddedToken(a_     ,   lstrip=a_     ,   rstrip=a_ ) if isinstance(a_     ,   a_ ) else bos_token
      UpperCAmelCase_						:    Dict    					=  AddedToken(a_     ,   lstrip=a_     ,   rstrip=a_ ) if isinstance(a_     ,   a_ ) else eos_token
      UpperCAmelCase_						:    Optional[int]    					=  AddedToken(a_     ,   lstrip=a_     ,   rstrip=a_ ) if isinstance(a_     ,   a_ ) else unk_token
      UpperCAmelCase_						:    int    					=  AddedToken(a_     ,   lstrip=a_     ,   rstrip=a_ ) if isinstance(a_     ,   a_ ) else pad_token
      super().__init__(
          errors=a_     ,   unk_token=a_     ,   bos_token=a_     ,   eos_token=a_     ,   pad_token=a_     ,   add_prefix_space=a_     ,   add_bos_token=a_     ,   **a_     ,   )
      UpperCAmelCase_						:    List[Any]    					=  add_bos_token

      with open(a_     ,   encoding="""utf-8""" ) as vocab_handle:
           UpperCAmelCase_						:    Dict    					=  json.load(a_ )
      UpperCAmelCase_						:    Any    					=  {v: k for k, v in self.encoder.items()}
      UpperCAmelCase_						:    Union[str, Any]    					=  errors  # how to handle errors in decoding
      UpperCAmelCase_						:    Dict    					=  bytes_to_unicode()
      UpperCAmelCase_						:    List[Any]    					=  {v: k for k, v in self.byte_encoder.items()}
      with open(a_     ,   encoding="""utf-8""" ) as merges_handle:
           UpperCAmelCase_						:    Union[str, Any]    					=  merges_handle.read().split("""\n""" )[1:-1]
      UpperCAmelCase_						:    List[str]    					=  [tuple(merge.split() ) for merge in bpe_merges]
      UpperCAmelCase_						:    Any    					=  dict(zip(a_     ,   range(len(a_ ) ) ) )
      UpperCAmelCase_						:    int    					=  {}
      UpperCAmelCase_						:    List[Any]    					=  add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      UpperCAmelCase_						:    str    					=  re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" )




 @property
 def   a      ( self       : Union[str, Any] )->					str:



      """simple docstring"""

      return len(self.encoder )




 def   a      ( self       : Dict )->					Dict:



      """simple docstring"""

      return dict(self.encoder     ,   **self.added_tokens_encoder )




 def   a      ( self       : List[str]     ,   a_       : Optional[int] )->					Optional[int]:



      """simple docstring"""

      if token in self.cache:
           return self.cache[token]
      UpperCAmelCase_						:    Optional[int]    					=  tuple(a_ )
      UpperCAmelCase_						:    int    					=  get_pairs(a_ )

      if not pairs:
           return token

      while True:
           UpperCAmelCase_						:    Union[str, Any]    					=  min(a_     ,   key=lambda a_ : self.bpe_ranks.get(a_     ,   float("""inf""" ) ) )
           if bigram not in self.bpe_ranks:
                break
           UpperCAmelCase_					,UpperCAmelCase_						:    Union[str, Any]    					=  bigram
           UpperCAmelCase_						:    List[str]    					=  []
           UpperCAmelCase_						:    List[str]    					=  0
           while i < len(a_ ):
                try:
                     UpperCAmelCase_						:    Tuple    					=  word.index(a_     ,   a_ )
                except ValueError:
                     new_word.extend(word[i:] )
                     break
                else:
                     new_word.extend(word[i:j] )
                     UpperCAmelCase_						:    Optional[int]    					=  j

                if word[i] == first and i < len(a_ ) - 1 and word[i + 1] == second:
                     new_word.append(first + second )
                     i += 2
                else:
                     new_word.append(word[i] )
                     i += 1
           UpperCAmelCase_						:    Optional[Any]    					=  tuple(a_ )
           UpperCAmelCase_						:    Any    					=  new_word
           if len(a_ ) == 1:
                break
           else:
                UpperCAmelCase_						:    List[Any]    					=  get_pairs(a_ )
      UpperCAmelCase_						:    Optional[int]    					=  """ """.join(a_ )
      UpperCAmelCase_						:    List[Any]    					=  word
      return word




 def   a      ( self       : Union[str, Any]     ,   a_       : Tuple     ,   a_       : List[Any]=None )->					List[Any]:



      """simple docstring"""

      if self.add_bos_token:
           UpperCAmelCase_						:    Union[str, Any]    					=  [self.bos_token_id]
      else:
           UpperCAmelCase_						:    Dict    					=  []

      UpperCAmelCase_						:    List[Any]    					=  bos_token_ids + token_ids_a

      if token_ids_a is None:
           return output

      return output + bos_token_ids + token_ids_a




 def   a      ( self       : Optional[int]     ,   a_       : List[int]     ,   a_       : Optional[List[int]] = None     ,   a_       : bool = False )->					List[int]:



      """simple docstring"""

      if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=a_     ,   token_ids_a=a_     ,   already_has_special_tokens=a_ )

      if not self.add_bos_token:
           return super().get_special_tokens_mask(
               token_ids_a=a_     ,   token_ids_a=a_     ,   already_has_special_tokens=a_ )

      if token_ids_a is None:
           return [1] + ([0] * len(a_ ))
      return [1] + ([0] * len(a_ )) + [1] + ([0] * len(a_ ))




 def   a      ( self       : Dict     ,   a_       : Dict )->					List[str]:



      """simple docstring"""

      UpperCAmelCase_						:    List[Any]    					=  []
      for token in re.findall(self.pat     ,   a_ ):
           UpperCAmelCase_						:    str    					=  """""".join(
               self.byte_encoder[b] for b in token.encode("""utf-8""" ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
           bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a_ ).split(""" """ ) )
      return bpe_tokens




 def   a      ( self       : List[str]     ,   a_       : int )->					Tuple:



      """simple docstring"""

      return self.encoder.get(a_     ,   self.encoder.get(self.unk_token ) )




 def   a      ( self       : Dict     ,   a_       : List[str] )->					Optional[Any]:



      """simple docstring"""

      return self.decoder.get(a_ )




 def   a      ( self       : Union[str, Any]     ,   a_       : Optional[int] )->					Optional[Any]:



      """simple docstring"""

      UpperCAmelCase_						:    List[str]    					=  """""".join(a_ )
      UpperCAmelCase_						:    Optional[Any]    					=  bytearray([self.byte_decoder[c] for c in text] ).decode("""utf-8"""     ,   errors=self.errors )
      return text




 def   a      ( self       : int     ,   a_       : str     ,   a_       : Optional[str] = None )->					Tuple[str]:



      """simple docstring"""

      if not os.path.isdir(a_ ):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
           return
      UpperCAmelCase_						:    Optional[Any]    					=  os.path.join(
          a_     ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )
      UpperCAmelCase_						:    Dict    					=  os.path.join(
          a_     ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""] )

      with open(a_     ,   """w"""     ,   encoding="""utf-8""" ) as f:
           f.write(json.dumps(self.encoder     ,   indent=2     ,   sort_keys=a_     ,   ensure_ascii=a_ ) + """\n""" )

      UpperCAmelCase_						:    List[Any]    					=  0
      with open(a_     ,   """w"""     ,   encoding="""utf-8""" ) as writer:
           writer.write("""#version: 0.2\n""" )
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,   key=lambda a_ : kv[1] ):
                if index != token_index:
                     logger.warning(
                         f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                         """ Please check that the tokenizer is not corrupted!""" )
                     UpperCAmelCase_						:    Dict    					=  token_index
                writer.write(""" """.join(a_ ) + """\n""" )
                index += 1

      return vocab_file, merge_file




 def   a      ( self       : Union[str, Any]     ,   a_       : Optional[int]     ,   a_       : List[Any]=False     ,   **a_       : Tuple )->					Optional[Any]:



      """simple docstring"""

      UpperCAmelCase_						:    int    					=  kwargs.pop("""add_prefix_space"""     ,   self.add_prefix_space )
      if is_split_into_words or add_prefix_space:
           UpperCAmelCase_						:    Any    					=  """ """ + text
      return (text, kwargs)






 def   a      ( self       : str     ,   a_       : "Conversation" )->					List[int]:



      """simple docstring"""

      UpperCAmelCase_						:    Optional[int]    					=  []
      for is_user, text in conversation.iter_texts():
           input_ids.extend(self.encode(a_     ,   add_special_tokens=a_ ) + [self.eos_token_id] )
      if len(a_ ) > self.model_max_length:
           UpperCAmelCase_						:    Optional[int]    					=  input_ids[-self.model_max_length :]
      return input_ids



"""simple docstring"""







import warnings

from .generation import GenerationMixin



class 		UpperCAmelCase_	(lowerCamelCase_						):
 """simple docstring"""




 warnings.warn(
     """Importing `GenerationMixin` from `src/transformers/generation_utils.py` is deprecated and will """
     """be removed in Transformers v5. Import as `from transformers import GenerationMixin` instead."""  , lowerCamelCase_  , )

