

import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_a	 =     logging.get_logger(__name__)

_a	 =     {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_a	 =     {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

_a	 =     {
    'Salesforce/codegen-350M-mono': 2_0_4_8,
}







@lru_cache()
def _a				( )					->					Optional[int]:

 """simple docstring"""






 __lowerCAmelCase:     Dict     			=   (
     list(range(ord('!'	)					,      ord('~'	) + 1	)	) + list(range(ord('¡'	)					,      ord('¬'	) + 1	)	) + list(range(ord('®'	)					,      ord('ÿ'	) + 1	)	)
 )
 __lowerCAmelCase:     Tuple     			=   bs[:]
 __lowerCAmelCase:     Dict     			=   0
 for b in range(2**8	):
  if b not in bs:
   bs.append(SCREAMING_SNAKE_CASE	)
   cs.append(2**8 + n	)
   n += 1
 __lowerCAmelCase:     str     			=   [chr(SCREAMING_SNAKE_CASE	) for n in cs]
 return dict(zip(SCREAMING_SNAKE_CASE					,      SCREAMING_SNAKE_CASE	)	)







def _a				( SCREAMING_SNAKE_CASE	:    Any	)					->					Tuple:

 """simple docstring"""






 __lowerCAmelCase:     Optional[int]     			=   set()
 __lowerCAmelCase:     Optional[Any]     			=   word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)	)
  __lowerCAmelCase:     Dict     			=   char
 return pairs






class 							A_				(	lowercase_      ):
     _lowercase    :  int          =			VOCAB_FILES_NAMES
     _lowercase    :  Union[str, Any]          =			PRETRAINED_VOCAB_FILES_MAP
     _lowercase    :  Optional[int]          =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowercase    :  Tuple          =			["""input_ids""", """attention_mask"""]


     def __init__(							self    :	List[Any]    ,						UpperCAmelCase    :	List[str]    ,						UpperCAmelCase    :	List[Any]    ,						UpperCAmelCase    :	Optional[int]="replace"    ,						UpperCAmelCase    :	Optional[Any]="<|endoftext|>"    ,						UpperCAmelCase    :	str="<|endoftext|>"    ,						UpperCAmelCase    :	Optional[int]="<|endoftext|>"    ,						UpperCAmelCase    :	Optional[int]=None    ,						UpperCAmelCase    :	str=False    ,						UpperCAmelCase    :	Any=False    ,						**UpperCAmelCase    :	Dict    ,						)							->			List[str]:
      __lowerCAmelCase:     List[Any]     			=   AddedToken(UpperCAmelCase    ,						lstrip=UpperCAmelCase    ,						rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,						UpperCAmelCase      ) else bos_token
      __lowerCAmelCase:     Optional[Any]     			=   AddedToken(UpperCAmelCase    ,						lstrip=UpperCAmelCase    ,						rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,						UpperCAmelCase      ) else eos_token
      __lowerCAmelCase:     str     			=   AddedToken(UpperCAmelCase    ,						lstrip=UpperCAmelCase    ,						rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,						UpperCAmelCase      ) else unk_token
      __lowerCAmelCase:     Union[str, Any]     			=   AddedToken(UpperCAmelCase    ,						lstrip=UpperCAmelCase    ,						rstrip=UpperCAmelCase      ) if isinstance(UpperCAmelCase    ,						UpperCAmelCase      ) else pad_token
      super().__init__(
          errors=UpperCAmelCase    ,						unk_token=UpperCAmelCase    ,						bos_token=UpperCAmelCase    ,						eos_token=UpperCAmelCase    ,						pad_token=UpperCAmelCase    ,						add_prefix_space=UpperCAmelCase    ,						add_bos_token=UpperCAmelCase    ,						**UpperCAmelCase    ,						)
      __lowerCAmelCase:     Any     			=   add_bos_token

      with open(UpperCAmelCase    ,						encoding='utf-8'      ) as vocab_handle:
       __lowerCAmelCase:     Optional[Any]     			=   json.load(UpperCAmelCase      )
      __lowerCAmelCase:     Tuple     			=   {v: k for k, v in self.encoder.items()}
      __lowerCAmelCase:     Optional[int]     			=   errors  # how to handle errors in decoding
      __lowerCAmelCase:     Optional[Any]     			=   bytes_to_unicode()
      __lowerCAmelCase:     Optional[int]     			=   {v: k for k, v in self.byte_encoder.items()}
      with open(UpperCAmelCase    ,						encoding='utf-8'      ) as merges_handle:
       __lowerCAmelCase:     Union[str, Any]     			=   merges_handle.read().split('\n'      )[1:-1]
      __lowerCAmelCase:     Dict     			=   [tuple(merge.split()      ) for merge in bpe_merges]
      __lowerCAmelCase:     Optional[int]     			=   dict(zip(UpperCAmelCase    ,						range(len(UpperCAmelCase      )      )      )      )
      __lowerCAmelCase:     int     			=   {}
      __lowerCAmelCase:     Dict     			=   add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      __lowerCAmelCase:     Tuple     			=   re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'      )


     @property
     def 						UpperCAmelCase						(							self    :	Any      )							->			List[str]:
      return len(self.encoder      )


     def 						UpperCAmelCase						(							self    :	Optional[Any]      )							->			Optional[Any]:
      return dict(self.encoder    ,						**self.added_tokens_encoder      )


     def 						UpperCAmelCase						(							self    :	Union[str, Any]    ,						UpperCAmelCase    :	Optional[int]      )							->			Tuple:







      if token in self.cache:
       return self.cache[token]
      __lowerCAmelCase:     Optional[int]     			=   tuple(UpperCAmelCase      )
      __lowerCAmelCase:     Optional[Any]     			=   get_pairs(UpperCAmelCase      )

      if not pairs:
       return token

      while True:
       __lowerCAmelCase:     str     			=   min(UpperCAmelCase    ,						key=lambda UpperCAmelCase      : self.bpe_ranks.get(UpperCAmelCase    ,						float('inf'      )      )      )
       if bigram not in self.bpe_ranks:
        break
       __lowerCAmelCase			,							__lowerCAmelCase:     Optional[Any]     			=   bigram
       __lowerCAmelCase:     int     			=   []
       __lowerCAmelCase:     Dict     			=   0
       while i < len(UpperCAmelCase      ):
        try:
         __lowerCAmelCase:     Any     			=   word.index(UpperCAmelCase    ,						UpperCAmelCase      )
        except ValueError:
         new_word.extend(word[i:]      )
         break
        else:
         new_word.extend(word[i:j]      )
         __lowerCAmelCase:     Optional[Any]     			=   j

        if word[i] == first and i < len(UpperCAmelCase      ) - 1 and word[i + 1] == second:
         new_word.append(first + second      )
         i += 2
        else:
         new_word.append(word[i]      )
         i += 1
       __lowerCAmelCase:     str     			=   tuple(UpperCAmelCase      )
       __lowerCAmelCase:     List[Any]     			=   new_word
       if len(UpperCAmelCase      ) == 1:
        break
       else:
        __lowerCAmelCase:     Optional[Any]     			=   get_pairs(UpperCAmelCase      )
      __lowerCAmelCase:     Any     			=   ' '.join(UpperCAmelCase      )
      __lowerCAmelCase:     Optional[Any]     			=   word
      return word


     def 						UpperCAmelCase						(							self    :	Any    ,						UpperCAmelCase    :	str    ,						UpperCAmelCase    :	str=None      )							->			Union[str, Any]:







      if self.add_bos_token:
       __lowerCAmelCase:     List[str]     			=   [self.bos_token_id]
      else:
       __lowerCAmelCase:     int     			=   []

      __lowerCAmelCase:     List[Any]     			=   bos_token_ids + token_ids_a

      if token_ids_a is None:
       return output

      return output + bos_token_ids + token_ids_a


     def 						UpperCAmelCase						(							self    :	Optional[Any]    ,						UpperCAmelCase    :	str      )							->			List[Any]:
      __lowerCAmelCase:     Optional[Any]     			=   []
      for token in re.findall(self.pat    ,						UpperCAmelCase      ):
       __lowerCAmelCase:     Any     			=   ''.join(
           self.byte_encoder[b] for b in token.encode('utf-8'      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase      ).split(' '      )      )
      return bpe_tokens


     def 						UpperCAmelCase						(							self    :	str    ,						UpperCAmelCase    :	str      )							->			Any:
      return self.encoder.get(UpperCAmelCase    ,						self.encoder.get(self.unk_token      )      )


     def 						UpperCAmelCase						(							self    :	Any    ,						UpperCAmelCase    :	Optional[int]      )							->			List[Any]:
      return self.decoder.get(UpperCAmelCase      )


     def 						UpperCAmelCase						(							self    :	List[Any]    ,						UpperCAmelCase    :	Optional[int]      )							->			Optional[Any]:
      __lowerCAmelCase:     str     			=   ''.join(UpperCAmelCase      )
      __lowerCAmelCase:     int     			=   bytearray([self.byte_decoder[c] for c in text]      ).decode('utf-8'    ,						errors=self.errors      )
      return text


     def 						UpperCAmelCase						(							self    :	int    ,						UpperCAmelCase    :	str    ,						UpperCAmelCase    :	Optional[str] = None      )							->			Union[str, Any]:







      if not os.path.isdir(UpperCAmelCase      ):
       logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
       return
      __lowerCAmelCase:     Union[str, Any]     			=   os.path.join(
          UpperCAmelCase    ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
      __lowerCAmelCase:     Union[str, Any]     			=   os.path.join(
          UpperCAmelCase    ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

      with open(UpperCAmelCase    ,						'w'    ,						encoding='utf-8'      ) as f:
       f.write(json.dumps(self.encoder    ,						indent=2    ,						sort_keys=UpperCAmelCase    ,						ensure_ascii=UpperCAmelCase      ) + '\n'      )

      __lowerCAmelCase:     List[Any]     			=   0
      with open(UpperCAmelCase    ,						'w'    ,						encoding='utf-8'      ) as writer:
       writer.write('#version: 0.2\n'      )
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,						key=lambda UpperCAmelCase      : kv[1]      ):
        if index != token_index:
         logger.warning(
             F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
             ' Please check that the tokenizer is not corrupted!'      )
         __lowerCAmelCase:     int     			=   token_index
        writer.write(' '.join(UpperCAmelCase      ) + '\n'      )
        index += 1

      return vocab_file, merge_file


     def 						UpperCAmelCase						(							self    :	str    ,						UpperCAmelCase    :	List[Any]    ,						UpperCAmelCase    :	int=False    ,						**UpperCAmelCase    :	Optional[int]      )							->			Optional[int]:
      __lowerCAmelCase:     Any     			=   kwargs.pop('add_prefix_space'    ,						self.add_prefix_space      )
      if is_split_into_words or add_prefix_space:
       __lowerCAmelCase:     List[Any]     			=   ' ' + text
      return (text, kwargs)


     def 						UpperCAmelCase						(							self    :	Tuple    ,						UpperCAmelCase    :	Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]    ,						UpperCAmelCase    :	bool = False    ,						UpperCAmelCase    :	bool = None    ,						UpperCAmelCase    :	Optional[List[str]] = None    ,						**UpperCAmelCase    :	Optional[Any]    ,						)							->			List[str]:
      __lowerCAmelCase:     List[str]     			=   super()._decode(
          token_ids=UpperCAmelCase    ,						skip_special_tokens=UpperCAmelCase    ,						clean_up_tokenization_spaces=UpperCAmelCase    ,						**UpperCAmelCase    ,						)

      if truncate_before_pattern is not None and len(UpperCAmelCase      ) > 0:
       __lowerCAmelCase:     Optional[int]     			=   self.truncate(UpperCAmelCase    ,						UpperCAmelCase      )

      return decoded_text


     def 						UpperCAmelCase						(							self    :	str    ,						UpperCAmelCase    :	List[str]    ,						UpperCAmelCase    :	Union[str, Any]      )							->			Dict:







      def find_re(UpperCAmelCase    :	Dict    ,						UpperCAmelCase    :	Dict    ,						UpperCAmelCase    :	Optional[Any]      ):
       __lowerCAmelCase:     str     			=   pattern.search(UpperCAmelCase    ,						UpperCAmelCase      )
       return m.start() if m else -1

      __lowerCAmelCase:     List[str]     			=   [re.compile(UpperCAmelCase    ,						re.MULTILINE      ) for pattern in truncate_before_pattern]

      __lowerCAmelCase:     List[str]     			=   list(re.finditer('^print'    ,						UpperCAmelCase    ,						re.MULTILINE      )      )

      if len(UpperCAmelCase      ) > 1:
       __lowerCAmelCase:     Union[str, Any]     			=   completion[: prints[1].start()]

      __lowerCAmelCase:     str     			=   list(re.finditer('^def'    ,						UpperCAmelCase    ,						re.MULTILINE      )      )

      if len(UpperCAmelCase      ) > 1:
       __lowerCAmelCase:     int     			=   completion[: defs[1].start()]

      __lowerCAmelCase:     Tuple     			=   0

      __lowerCAmelCase:     Any     			=   [
          pos for pos in [find_re(UpperCAmelCase    ,						UpperCAmelCase    ,						UpperCAmelCase      ) for terminal in terminals] if pos != -1
      ]

      if len(UpperCAmelCase      ) > 0:
       return completion[: min(UpperCAmelCase      )]
      else:
       return completion








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_keras_nlp_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_a	 =     {
    '''configuration_gpt2''': ['''GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''GPT2Config''', '''GPT2OnnxConfig'''],
    '''tokenization_gpt2''': ['''GPT2Tokenizer'''],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _a	 =     ['''GPT2TokenizerFast''']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _a	 =     [
      '''GPT2_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''GPT2DoubleHeadsModel''',
      '''GPT2ForQuestionAnswering''',
      '''GPT2ForSequenceClassification''',
      '''GPT2ForTokenClassification''',
      '''GPT2LMHeadModel''',
      '''GPT2Model''',
      '''GPT2PreTrainedModel''',
      '''load_tf_weights_in_gpt2''',
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _a	 =     [
      '''TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''TFGPT2DoubleHeadsModel''',
      '''TFGPT2ForSequenceClassification''',
      '''TFGPT2LMHeadModel''',
      '''TFGPT2MainLayer''',
      '''TFGPT2Model''',
      '''TFGPT2PreTrainedModel''',
  ]

try:
  if not is_keras_nlp_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _a	 =     ['''TFGPT2Tokenizer''']

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _a	 =     ['''FlaxGPT2LMHeadModel''', '''FlaxGPT2Model''', '''FlaxGPT2PreTrainedModel''']

if TYPE_CHECKING:
  from .configuration_gpta import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTaConfig, GPTaOnnxConfig
  from .tokenization_gpta import GPTaTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_gpta_fast import GPTaTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_gpta import (
        GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
        GPTaDoubleHeadsModel,
        GPTaForQuestionAnswering,
        GPTaForSequenceClassification,
        GPTaForTokenClassification,
        GPTaLMHeadModel,
        GPTaModel,
        GPTaPreTrainedModel,
        load_tf_weights_in_gpta,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_gpta import (
        TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFGPTaDoubleHeadsModel,
        TFGPTaForSequenceClassification,
        TFGPTaLMHeadModel,
        TFGPTaMainLayer,
        TFGPTaModel,
        TFGPTaPreTrainedModel,
    )

  try:
    if not is_keras_nlp_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_gpta_tf import TFGPTaTokenizer

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel, FlaxGPTaPreTrainedModel

else:
  import sys

  _a	 =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





