






'''simple docstring'''


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


SCREAMING_SNAKE_CASE_				      =		logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_				      =		{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

SCREAMING_SNAKE_CASE_				      =		{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
}

SCREAMING_SNAKE_CASE_				      =		{
    "Salesforce/codegen-350M-mono": 2_0_4_8,
}






@lru_cache()
def   lowerCAmelCase__			(					):
 __a    :	List[str]									=						(
     list(range(ord('!'		)       ,		ord('~'		) + 1		)		) + list(range(ord('¡'		)       ,		ord('¬'		) + 1		)		) + list(range(ord('®'		)       ,		ord('ÿ'		) + 1		)		)
 )
 __a    :	str									=						bs[:]
 __a    :	Optional[Any]									=						0
 for b in range(2**8		):
  if b not in bs:
   bs.append(SCREAMING_SNAKE_CASE__		)
   cs.append(2**8 + n		)
   n += 1
 __a    :	Union[str, Any]									=						[chr(SCREAMING_SNAKE_CASE__		) for n in cs]
 return dict(zip(SCREAMING_SNAKE_CASE__       ,		SCREAMING_SNAKE_CASE__		)		)






def   lowerCAmelCase__			(					SCREAMING_SNAKE_CASE__		):
 __a    :	Tuple									=						set()
 __a    :	Dict									=						word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)		)
  __a    :	str									=						char
 return pairs




class 							lowerCAmelCase					(							lowerCamelCase__   ):







   """simple docstring"""







   _A						=   VOCAB_FILES_NAMES
   _A						=   PRETRAINED_VOCAB_FILES_MAP
   _A						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _A						=   ['input_ids', 'attention_mask']



   def __init__( self   ,   _A   ,   _A   ,   _A="replace"   ,   _A="<|endoftext|>"   ,   _A="<|endoftext|>"   ,   _A="<|endoftext|>"   ,   _A=None   ,   _A=False   ,   _A=False   ,   **_A   ,   )    ->   Optional[Any]:
    __a    :	List[str]									=						AddedToken(_A   ,   lstrip=_A   ,   rstrip=_A			) if isinstance(_A   ,   _A			) else bos_token
    __a    :	Dict									=						AddedToken(_A   ,   lstrip=_A   ,   rstrip=_A			) if isinstance(_A   ,   _A			) else eos_token
    __a    :	Optional[Any]									=						AddedToken(_A   ,   lstrip=_A   ,   rstrip=_A			) if isinstance(_A   ,   _A			) else unk_token
    __a    :	List[Any]									=						AddedToken(_A   ,   lstrip=_A   ,   rstrip=_A			) if isinstance(_A   ,   _A			) else pad_token
    super().__init__(
        errors=_A   ,   unk_token=_A   ,   bos_token=_A   ,   eos_token=_A   ,   pad_token=_A   ,   add_prefix_space=_A   ,   add_bos_token=_A   ,   **_A   ,   )
    __a    :	Any									=						add_bos_token

    with open(_A   ,   encoding='utf-8'			) as vocab_handle:
     __a    :	Optional[Any]									=						json.load(_A			)
    __a    :	List[str]									=						{v: k for k, v in self.encoder.items()}
    __a    :	Optional[Any]									=						errors  # how to handle errors in decoding
    __a    :	Union[str, Any]									=						bytes_to_unicode()
    __a    :	List[str]									=						{v: k for k, v in self.byte_encoder.items()}
    with open(_A   ,   encoding='utf-8'			) as merges_handle:
     __a    :	Union[str, Any]									=						merges_handle.read().split('\n'			)[1:-1]
    __a    :	List[str]									=						[tuple(merge.split()			) for merge in bpe_merges]
    __a    :	Optional[int]									=						dict(zip(_A   ,   range(len(_A			)			)			)			)
    __a    :	List[str]									=						{}
    __a    :	Tuple									=						add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    __a    :	int									=						re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'			)



   @property
   def    __magic_name__    ( self			)    ->   List[str]:
    return len(self.encoder			)



   def    __magic_name__    ( self			)    ->   Optional[int]:
    return dict(self.encoder   ,   **self.added_tokens_encoder			)



   def    __magic_name__    ( self   ,   _A			)    ->   Any:
    if token in self.cache:
     return self.cache[token]
    __a    :	int									=						tuple(_A			)
    __a    :	int									=						get_pairs(_A			)

    if not pairs:
     return token

    while True:
     __a    :	Optional[Any]									=						min(_A   ,   key=lambda _A			: self.bpe_ranks.get(_A   ,   float('inf'			)			)			)
     if bigram not in self.bpe_ranks:
      break
     __a    :	Optional[int]									=						bigram
     __a    :	Optional[int]									=						[]
     __a    :	Dict									=						0
     while i < len(_A			):
      try:
       __a    :	Any									=						word.index(_A   ,   _A			)
      except ValueError:
       new_word.extend(word[i:]			)
       break
      else:
       new_word.extend(word[i:j]			)
       __a    :	Dict									=						j

      if word[i] == first and i < len(_A			) - 1 and word[i + 1] == second:
       new_word.append(first + second			)
       i += 2
      else:
       new_word.append(word[i]			)
       i += 1
     __a    :	List[str]									=						tuple(_A			)
     __a    :	Any									=						new_word
     if len(_A			) == 1:
      break
     else:
      __a    :	str									=						get_pairs(_A			)
    __a    :	Tuple									=						' '.join(_A			)
    __a    :	Dict									=						word
    return word



   def    __magic_name__    ( self   ,   _A   ,   _A=None			)    ->   int:
    if self.add_bos_token:
     __a    :	Tuple									=						[self.bos_token_id]
    else:
     __a    :	List[str]									=						[]

    __a    :	Union[str, Any]									=						bos_token_ids + token_ids_a

    if token_ids_a is None:
     return output

    return output + bos_token_ids + token_ids_a



   def    __magic_name__    ( self   ,   _A			)    ->   Optional[int]:
    __a    :	int									=						[]
    for token in re.findall(self.pat   ,   _A			):
     __a    :	Union[str, Any]									=						''.join(
         self.byte_encoder[b] for b in token.encode('utf-8'			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
     bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_A			).split(' '			)			)
    return bpe_tokens



   def    __magic_name__    ( self   ,   _A			)    ->   Tuple:
    return self.encoder.get(_A   ,   self.encoder.get(self.unk_token			)			)



   def    __magic_name__    ( self   ,   _A			)    ->   List[str]:
    return self.decoder.get(_A			)



   def    __magic_name__    ( self   ,   _A			)    ->   List[Any]:
    __a    :	Dict									=						''.join(_A			)
    __a    :	int									=						bytearray([self.byte_decoder[c] for c in text]			).decode('utf-8'   ,   errors=self.errors			)
    return text



   def    __magic_name__    ( self   ,   _A   ,   _A = None			)    ->   Tuple[str]:
    if not os.path.isdir(_A			):
     logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''			)
     return
    __a    :	int									=						os.path.join(
        _A   ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)
    __a    :	List[str]									=						os.path.join(
        _A   ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']			)

    with open(_A   ,   'w'   ,   encoding='utf-8'			) as f:
     f.write(json.dumps(self.encoder   ,   indent=2   ,   sort_keys=_A   ,   ensure_ascii=_A			) + '\n'			)

    __a    :	str									=						0
    with open(_A   ,   'w'   ,   encoding='utf-8'			) as writer:
     writer.write('#version: 0.2\n'			)
     for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,   key=lambda _A			: kv[1]			):
      if index != token_index:
       logger.warning(
           f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
           ' Please check that the tokenizer is not corrupted!'			)
       __a    :	Optional[Any]									=						token_index
      writer.write(' '.join(_A			) + '\n'			)
      index += 1

    return vocab_file, merge_file



   def    __magic_name__    ( self   ,   _A   ,   _A=False   ,   **_A			)    ->   str:
    __a    :	int									=						kwargs.pop('add_prefix_space'   ,   self.add_prefix_space			)
    if is_split_into_words or add_prefix_space:
     __a    :	Optional[int]									=						' ' + text
    return (text, kwargs)



   def    __magic_name__    ( self   ,   _A   ,   _A = False   ,   _A = None   ,   _A = None   ,   **_A   ,   )    ->   str:
    __a    :	Dict									=						super()._decode(
        token_ids=_A   ,   skip_special_tokens=_A   ,   clean_up_tokenization_spaces=_A   ,   **_A   ,   )

    if truncate_before_pattern is not None and len(_A			) > 0:
     __a    :	Optional[int]									=						self.truncate(_A   ,   _A			)

    return decoded_text



   def    __magic_name__    ( self   ,   _A   ,   _A			)    ->   Optional[int]:
    def find_re(_A   ,   _A   ,   _A			):
     __a    :	Optional[Any]									=						pattern.search(_A   ,   _A			)
     return m.start() if m else -1

    __a    :	Tuple									=						[re.compile(_A   ,   re.MULTILINE			) for pattern in truncate_before_pattern]

    __a    :	Union[str, Any]									=						list(re.finditer('^print'   ,   _A   ,   re.MULTILINE			)			)

    if len(_A			) > 1:
     __a    :	Union[str, Any]									=						completion[: prints[1].start()]

    __a    :	str									=						list(re.finditer('^def'   ,   _A   ,   re.MULTILINE			)			)

    if len(_A			) > 1:
     __a    :	List[str]									=						completion[: defs[1].start()]

    __a    :	List[str]									=						0

    __a    :	List[str]									=						[
        pos for pos in [find_re(_A   ,   _A   ,   _A			) for terminal in terminals] if pos != -1
    ]

    if len(_A			) > 0:
     return completion[: min(_A			)]
    else:
     return completion












'''simple docstring'''


def   lowerCAmelCase__			(					SCREAMING_SNAKE_CASE__ = 30		):
 __a    :	Tuple									=						exponent + 2
 __a    :	str									=						(1 + 5**0.5) / 2
 __a    :	Any									=						(phi**fibonacci_index - (phi - 1) ** fibonacci_index) / 5**0.5

 return int(SCREAMING_SNAKE_CASE__		)


if __name__ == "__main__":
  print(F"{solution() = }")




