



'''simple docstring'''
import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
					if is_torch_available():
										import torch
					if is_tf_available():
										import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_UpperCamelCase											=			logging.get_logger(__name__)

_UpperCamelCase											=			{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_UpperCamelCase											=			{
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

_UpperCamelCase											=			{
    '''Salesforce/codegen-350M-mono''': 2048,
}


@lru_cache()
def   lowercase_	(   ):

			"""simple docstring"""





			__UpperCAmelCase   :      Dict										=     (
			    list(range(ord("""!""" )			,							ord("""~""" ) + 1 ) ) + list(range(ord("""¡""" )			,							ord("""¬""" ) + 1 ) ) + list(range(ord("""®""" )			,							ord("""ÿ""" ) + 1 ) )
			)
			__UpperCAmelCase   :      int										=     bs[:]
			__UpperCAmelCase   :      Optional[Any]										=     0
			for b in range(2**8 ):
						if b not in bs:
									bs.append(lowerCAmelCase__ )
									cs.append(2**8 + n )
									n += 1
			__UpperCAmelCase   :      Any										=     [chr(lowerCAmelCase__ ) for n in cs]
			return dict(zip(lowerCAmelCase__			,							lowerCAmelCase__ ) )


def   lowercase_	(   lowerCAmelCase__	:  Tuple ):

			"""simple docstring"""





			__UpperCAmelCase   :      Optional[Any]										=     set()
			__UpperCAmelCase   :      List[str]										=     word[0]
			for char in word[1:]:
						pairs.add((prev_char, char) )
						__UpperCAmelCase   :      Tuple										=     char
			return pairs






class _A  (    __SCREAMING_SNAKE_CASE    ):
		_SCREAMING_SNAKE_CASE					:    Union[str, Any]						      =			VOCAB_FILES_NAMES
		_SCREAMING_SNAKE_CASE					:    Tuple						      =			PRETRAINED_VOCAB_FILES_MAP
		_SCREAMING_SNAKE_CASE					:    Union[str, Any]						      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_SCREAMING_SNAKE_CASE					:    Optional[Any]						      =			["input_ids", "attention_mask"]





		def __init__(     self ,       __UpperCAmelCase ,       __UpperCAmelCase ,       __UpperCAmelCase="replace" ,       __UpperCAmelCase="<|endoftext|>" ,       __UpperCAmelCase="<|endoftext|>" ,       __UpperCAmelCase="<|endoftext|>" ,       __UpperCAmelCase=None ,       __UpperCAmelCase=False ,       __UpperCAmelCase=False ,       **__UpperCAmelCase ,       )       ->     Tuple:

					'''simple docstring'''
					__UpperCAmelCase   :      Dict										=     AddedToken(__UpperCAmelCase ,       lstrip=__UpperCAmelCase ,       rstrip=__UpperCAmelCase				) if isinstance(__UpperCAmelCase ,       __UpperCAmelCase				) else bos_token
					__UpperCAmelCase   :      Any										=     AddedToken(__UpperCAmelCase ,       lstrip=__UpperCAmelCase ,       rstrip=__UpperCAmelCase				) if isinstance(__UpperCAmelCase ,       __UpperCAmelCase				) else eos_token
					__UpperCAmelCase   :      int										=     AddedToken(__UpperCAmelCase ,       lstrip=__UpperCAmelCase ,       rstrip=__UpperCAmelCase				) if isinstance(__UpperCAmelCase ,       __UpperCAmelCase				) else unk_token
					__UpperCAmelCase   :      int										=     AddedToken(__UpperCAmelCase ,       lstrip=__UpperCAmelCase ,       rstrip=__UpperCAmelCase				) if isinstance(__UpperCAmelCase ,       __UpperCAmelCase				) else pad_token
					super().__init__(
					    errors=__UpperCAmelCase ,       unk_token=__UpperCAmelCase ,       bos_token=__UpperCAmelCase ,       eos_token=__UpperCAmelCase ,       pad_token=__UpperCAmelCase ,       add_prefix_space=__UpperCAmelCase ,       add_bos_token=__UpperCAmelCase ,       **__UpperCAmelCase ,       )
					__UpperCAmelCase   :      Optional[int]										=     add_bos_token

					with open(__UpperCAmelCase ,       encoding="""utf-8"""				) as vocab_handle:
								__UpperCAmelCase   :      Dict										=     json.load(__UpperCAmelCase				)
					__UpperCAmelCase   :      Optional[Any]										=     {v: k for k, v in self.encoder.items()}
					__UpperCAmelCase   :      int										=     errors  # how to handle errors in decoding
					__UpperCAmelCase   :      Dict										=     bytes_to_unicode()
					__UpperCAmelCase   :      Union[str, Any]										=     {v: k for k, v in self.byte_encoder.items()}
					with open(__UpperCAmelCase ,       encoding="""utf-8"""				) as merges_handle:
								__UpperCAmelCase   :      List[str]										=     merges_handle.read().split("""\n"""				)[1:-1]
					__UpperCAmelCase   :      str										=     [tuple(merge.split()				) for merge in bpe_merges]
					__UpperCAmelCase   :      Optional[int]										=     dict(zip(__UpperCAmelCase ,       range(len(__UpperCAmelCase				)				)				)				)
					__UpperCAmelCase   :      Dict										=     {}
					__UpperCAmelCase   :      Union[str, Any]										=     add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					__UpperCAmelCase   :      List[str]										=     re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""				)





		@property
		def 							__A       (     self				)       ->     List[Any]:

					'''simple docstring'''
					return len(self.encoder				)





		def 							__A       (     self				)       ->     Optional[Any]:

					'''simple docstring'''
					return dict(self.encoder ,       **self.added_tokens_encoder				)





		def 							__A       (     self ,       __UpperCAmelCase				)       ->     Dict:

					'''simple docstring'''
					if token in self.cache:
								return self.cache[token]
					__UpperCAmelCase   :      Optional[Any]										=     tuple(__UpperCAmelCase				)
					__UpperCAmelCase   :      Tuple										=     get_pairs(__UpperCAmelCase				)

					if not pairs:
								return token

					while True:
								__UpperCAmelCase   :      Dict										=     min(__UpperCAmelCase ,       key=lambda __UpperCAmelCase				: self.bpe_ranks.get(__UpperCAmelCase ,       float("""inf"""				)				)				)
								if bigram not in self.bpe_ranks:
											break
								__UpperCAmelCase				,  __UpperCAmelCase   :      str										=     bigram
								__UpperCAmelCase   :      int										=     []
								__UpperCAmelCase   :      Tuple										=     0
								while i < len(__UpperCAmelCase				):
											try:
														__UpperCAmelCase   :      str										=     word.index(__UpperCAmelCase ,       __UpperCAmelCase				)
											except ValueError:
														new_word.extend(word[i:]				)
														break
											else:
														new_word.extend(word[i:j]				)
														__UpperCAmelCase   :      Tuple										=     j

											if word[i] == first and i < len(__UpperCAmelCase				) - 1 and word[i + 1] == second:
														new_word.append(first + second				)
														i += 2
											else:
														new_word.append(word[i]				)
														i += 1
								__UpperCAmelCase   :      Optional[Any]										=     tuple(__UpperCAmelCase				)
								__UpperCAmelCase   :      List[Any]										=     new_word
								if len(__UpperCAmelCase				) == 1:
											break
								else:
											__UpperCAmelCase   :      Tuple										=     get_pairs(__UpperCAmelCase				)
					__UpperCAmelCase   :      Optional[int]										=     """ """.join(__UpperCAmelCase				)
					__UpperCAmelCase   :      Optional[int]										=     word
					return word





		def 							__A       (     self ,       __UpperCAmelCase ,       __UpperCAmelCase=None				)       ->     int:

					'''simple docstring'''
					if self.add_bos_token:
								__UpperCAmelCase   :      List[Any]										=     [self.bos_token_id]
					else:
								__UpperCAmelCase   :      str										=     []

					__UpperCAmelCase   :      Any										=     bos_token_ids + token_ids_a

					if token_ids_a is None:
								return output

					return output + bos_token_ids + token_ids_a





		def 							__A       (     self ,       __UpperCAmelCase				)       ->     Optional[Any]:

					'''simple docstring'''
					__UpperCAmelCase   :      Tuple										=     []
					for token in re.findall(self.pat ,       __UpperCAmelCase				):
								__UpperCAmelCase   :      List[Any]										=     """""".join(
								    self.byte_encoder[b] for b in token.encode("""utf-8"""				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
								bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__UpperCAmelCase				).split(""" """				)				)
					return bpe_tokens





		def 							__A       (     self ,       __UpperCAmelCase				)       ->     str:

					'''simple docstring'''
					return self.encoder.get(__UpperCAmelCase ,       self.encoder.get(self.unk_token				)				)





		def 							__A       (     self ,       __UpperCAmelCase				)       ->     List[Any]:

					'''simple docstring'''
					return self.decoder.get(__UpperCAmelCase				)





		def 							__A       (     self ,       __UpperCAmelCase				)       ->     Dict:

					'''simple docstring'''
					__UpperCAmelCase   :      str										=     """""".join(__UpperCAmelCase				)
					__UpperCAmelCase   :      Optional[Any]										=     bytearray([self.byte_decoder[c] for c in text]				).decode("""utf-8""" ,       errors=self.errors				)
					return text





		def 							__A       (     self ,       __UpperCAmelCase ,       __UpperCAmelCase = None				)       ->     Tuple[str]:

					'''simple docstring'''
					if not os.path.isdir(__UpperCAmelCase				):
								logger.error(f'Vocabulary path ({save_directory}) should be a directory'				)
								return
					__UpperCAmelCase   :      List[Any]										=     os.path.join(
					    __UpperCAmelCase ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
					__UpperCAmelCase   :      Any										=     os.path.join(
					    __UpperCAmelCase ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]				)

					with open(__UpperCAmelCase ,       """w""" ,       encoding="""utf-8"""				) as f:
								f.write(json.dumps(self.encoder ,       indent=2 ,       sort_keys=__UpperCAmelCase ,       ensure_ascii=__UpperCAmelCase				) + """\n"""				)

					__UpperCAmelCase   :      Union[str, Any]										=     0
					with open(__UpperCAmelCase ,       """w""" ,       encoding="""utf-8"""				) as writer:
								writer.write("""#version: 0.2\n"""				)
								for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,       key=lambda __UpperCAmelCase				: kv[1]				):
											if index != token_index:
														logger.warning(
														    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
														    """ Please check that the tokenizer is not corrupted!"""				)
														__UpperCAmelCase   :      Union[str, Any]										=     token_index
											writer.write(""" """.join(__UpperCAmelCase				) + """\n"""				)
											index += 1

					return vocab_file, merge_file





		def 							__A       (     self ,       __UpperCAmelCase ,       __UpperCAmelCase=False ,       **__UpperCAmelCase				)       ->     List[str]:

					'''simple docstring'''
					__UpperCAmelCase   :      Optional[Any]										=     kwargs.pop("""add_prefix_space""" ,       self.add_prefix_space				)
					if is_split_into_words or add_prefix_space:
								__UpperCAmelCase   :      int										=     """ """ + text
					return (text, kwargs)





		def 							__A       (     self ,       __UpperCAmelCase ,       __UpperCAmelCase = False ,       __UpperCAmelCase = None ,       __UpperCAmelCase = None ,       **__UpperCAmelCase ,       )       ->     str:

					'''simple docstring'''
					__UpperCAmelCase   :      List[str]										=     super()._decode(
					    token_ids=__UpperCAmelCase ,       skip_special_tokens=__UpperCAmelCase ,       clean_up_tokenization_spaces=__UpperCAmelCase ,       **__UpperCAmelCase ,       )

					if truncate_before_pattern is not None and len(__UpperCAmelCase				) > 0:
								__UpperCAmelCase   :      Union[str, Any]										=     self.truncate(__UpperCAmelCase ,       __UpperCAmelCase				)

					return decoded_text





		def 							__A       (     self ,       __UpperCAmelCase ,       __UpperCAmelCase				)       ->     List[Any]:

					'''simple docstring'''
					def find_re(__UpperCAmelCase ,       __UpperCAmelCase ,       __UpperCAmelCase				):
								__UpperCAmelCase   :      Union[str, Any]										=     pattern.search(__UpperCAmelCase ,       __UpperCAmelCase				)
								return m.start() if m else -1

					__UpperCAmelCase   :      Optional[int]										=     [re.compile(__UpperCAmelCase ,       re.MULTILINE				) for pattern in truncate_before_pattern]

					__UpperCAmelCase   :      List[str]										=     list(re.finditer("""^print""" ,       __UpperCAmelCase ,       re.MULTILINE				)				)

					if len(__UpperCAmelCase				) > 1:
								__UpperCAmelCase   :      Union[str, Any]										=     completion[: prints[1].start()]

					__UpperCAmelCase   :      Dict										=     list(re.finditer("""^def""" ,       __UpperCAmelCase ,       re.MULTILINE				)				)

					if len(__UpperCAmelCase				) > 1:
								__UpperCAmelCase   :      List[Any]										=     completion[: defs[1].start()]

					__UpperCAmelCase   :      List[str]										=     0

					__UpperCAmelCase   :      int										=     [
					    pos for pos in [find_re(__UpperCAmelCase ,       __UpperCAmelCase ,       __UpperCAmelCase				) for terminal in terminals] if pos != -1
					]

					if len(__UpperCAmelCase				) > 0:
								return completion[: min(__UpperCAmelCase				)]
					else:
								return completion









'''simple docstring'''
from __future__ import annotations

from typing import Any



class _A  :





		def __init__(     self ,       __UpperCAmelCase ,       __UpperCAmelCase				)       ->     None:  # noqa: A002

					'''simple docstring'''
					__UpperCAmelCase   :      Tuple										=     item
					__UpperCAmelCase   :      str										=     next



class _A  :





		def __init__(     self				)       ->     None:

					'''simple docstring'''
					__UpperCAmelCase   :      Node | None										=     None
					__UpperCAmelCase   :      Optional[int]										=     0





		def 							__A       (     self ,       __UpperCAmelCase				)       ->     None:

					'''simple docstring'''
					__UpperCAmelCase   :      Optional[Any]										=     Node(__UpperCAmelCase ,       self.head				)
					self.size += 1





		def 							__A       (     self				)       ->     Any:

					'''simple docstring'''
					# Switched 'self.is_empty()' to 'self.head is None'
					# because mypy was considering the possibility that 'self.head'
					# can be None in below else part and giving error
					if self.head is None:
								return None
					else:
								__UpperCAmelCase   :      str										=     self.head.item
								__UpperCAmelCase   :      Tuple										=     self.head.next
								self.size -= 1
								return item





		def 							__A       (     self				)       ->     bool:

					'''simple docstring'''
					return self.head is None





		def __str__(     self				)       ->     str:

					'''simple docstring'''
					if self.is_empty():
								return ""
					else:
								__UpperCAmelCase   :      Union[str, Any]										=     self.head
								__UpperCAmelCase   :      List[Any]										=     """"""
								__UpperCAmelCase   :      list[str]										=     []
								while iterate:
											item_list.append(str(iterate.item				)				)
											__UpperCAmelCase   :      Any										=     iterate.next

								__UpperCAmelCase   :      Dict										=     """ --> """.join(__UpperCAmelCase				)

								return item_str





		def __len__(     self				)       ->     int:

					'''simple docstring'''
					return self.size




