
"""simple docstring"""


import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A_						       =				logging.get_logger(__name__)

A_						       =				{
    """src_vocab_file""": """vocab-src.json""",
    """tgt_vocab_file""": """vocab-tgt.json""",
    """merges_file""": """merges.txt""",
}

A_						       =				{
    """src_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"""
    },
    """tgt_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"""
    },
    """merges_file""": {"""stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"""},
}

A_						       =				{"""stas/tiny-wmt19-en-de""": 1024}
A_						       =				{
    """stas/tiny-wmt19-en-de""": {
        """langs""": ["""en""", """de"""],
        """model_max_length""": 1024,
        """special_tokens_map_file""": None,
        """full_tokenizer_file""": None,
    }
}






def    lowercase	(  lowerCAmelCase__	):
			lowerCamelCase_ 		=     set()
			lowerCamelCase_ 		=     word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)	)
						lowerCamelCase_ 		=     char
			return pairs






def    lowercase	(  lowerCAmelCase__	):
			lowerCamelCase_ 		=     text.replace('''，'''  ,''','''	)
			lowerCamelCase_ 		=     re.sub(R'''。\s*'''  ,'''. '''  ,lowerCAmelCase__	)
			lowerCamelCase_ 		=     text.replace('''、'''  ,''','''	)
			lowerCamelCase_ 		=     text.replace('''”'''  ,'''"'''	)
			lowerCamelCase_ 		=     text.replace('''“'''  ,'''"'''	)
			lowerCamelCase_ 		=     text.replace('''∶'''  ,''':'''	)
			lowerCamelCase_ 		=     text.replace('''：'''  ,''':'''	)
			lowerCamelCase_ 		=     text.replace('''？'''  ,'''?'''	)
			lowerCamelCase_ 		=     text.replace('''《'''  ,'''"'''	)
			lowerCamelCase_ 		=     text.replace('''》'''  ,'''"'''	)
			lowerCamelCase_ 		=     text.replace('''）'''  ,''')'''	)
			lowerCamelCase_ 		=     text.replace('''！'''  ,'''!'''	)
			lowerCamelCase_ 		=     text.replace('''（'''  ,'''('''	)
			lowerCamelCase_ 		=     text.replace('''；'''  ,''';'''	)
			lowerCamelCase_ 		=     text.replace('''１'''  ,'''1'''	)
			lowerCamelCase_ 		=     text.replace('''」'''  ,'''"'''	)
			lowerCamelCase_ 		=     text.replace('''「'''  ,'''"'''	)
			lowerCamelCase_ 		=     text.replace('''０'''  ,'''0'''	)
			lowerCamelCase_ 		=     text.replace('''３'''  ,'''3'''	)
			lowerCamelCase_ 		=     text.replace('''２'''  ,'''2'''	)
			lowerCamelCase_ 		=     text.replace('''５'''  ,'''5'''	)
			lowerCamelCase_ 		=     text.replace('''６'''  ,'''6'''	)
			lowerCamelCase_ 		=     text.replace('''９'''  ,'''9'''	)
			lowerCamelCase_ 		=     text.replace('''７'''  ,'''7'''	)
			lowerCamelCase_ 		=     text.replace('''８'''  ,'''8'''	)
			lowerCamelCase_ 		=     text.replace('''４'''  ,'''4'''	)
			lowerCamelCase_ 		=     re.sub(R'''．\s*'''  ,'''. '''  ,lowerCAmelCase__	)
			lowerCamelCase_ 		=     text.replace('''～'''  ,'''~'''	)
			lowerCamelCase_ 		=     text.replace('''’'''  ,'''\''''	)
			lowerCamelCase_ 		=     text.replace('''…'''  ,'''...'''	)
			lowerCamelCase_ 		=     text.replace('''━'''  ,'''-'''	)
			lowerCamelCase_ 		=     text.replace('''〈'''  ,'''<'''	)
			lowerCamelCase_ 		=     text.replace('''〉'''  ,'''>'''	)
			lowerCamelCase_ 		=     text.replace('''【'''  ,'''['''	)
			lowerCamelCase_ 		=     text.replace('''】'''  ,''']'''	)
			lowerCamelCase_ 		=     text.replace('''％'''  ,'''%'''	)
			return text






def    lowercase	(  lowerCAmelCase__	):
			lowerCamelCase_ 		=     []
			for char in text:
						lowerCamelCase_ 		=     unicodedata.category(lowerCAmelCase__	)
						if cat.startswith('''C'''	):
									continue
						output.append(lowerCAmelCase__	)
			return "".join(lowerCAmelCase__	)




class    __lowerCamelCase     (   lowerCAmelCase ):
				a__:			Tuple									=						VOCAB_FILES_NAMES
				a__:			Optional[Any]									=						PRETRAINED_VOCAB_FILES_MAP
				a__:			Any									=						PRETRAINED_INIT_CONFIGURATION
				a__:			Tuple									=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				a__:			List[Any]									=						['input_ids', 'attention_mask']




				def __init__(							self       ,	UpperCAmelCase=None       ,	UpperCAmelCase=None       ,	UpperCAmelCase=None       ,	UpperCAmelCase=None       ,	UpperCAmelCase=False       ,	UpperCAmelCase="<unk>"       ,	UpperCAmelCase="<s>"       ,	UpperCAmelCase="</s>"       ,	UpperCAmelCase="<pad>"       ,	**UpperCAmelCase       ,	):
							super().__init__(
							    langs=UpperCAmelCase       ,	src_vocab_file=UpperCAmelCase       ,	tgt_vocab_file=UpperCAmelCase       ,	merges_file=UpperCAmelCase       ,	do_lower_case=UpperCAmelCase       ,	unk_token=UpperCAmelCase       ,	bos_token=UpperCAmelCase       ,	sep_token=UpperCAmelCase       ,	pad_token=UpperCAmelCase       ,	**UpperCAmelCase       ,	)

							try:
										import sacremoses
							except ImportError:
										raise ImportError(
										    '''You need to install sacremoses to use XLMTokenizer. '''
										    '''See https://pypi.org/project/sacremoses/ for installation.'''      )

							lowerCamelCase_ 		=     sacremoses

							lowerCamelCase_ 		=     src_vocab_file
							lowerCamelCase_ 		=     tgt_vocab_file
							lowerCamelCase_ 		=     merges_file
							lowerCamelCase_ 		=     do_lower_case

							# cache of sm.MosesPunctNormalizer instance
							lowerCamelCase_ 		=     {}
							# cache of sm.MosesTokenizer instance
							lowerCamelCase_ 		=     {}
							lowerCamelCase_ 		=     {}

							if langs and len(UpperCAmelCase      ) == 2:
										lowerCamelCase_		,   lowerCamelCase_ 		=     langs
							else:
										raise ValueError(
										    f"arg `langs` needs to be a list of 2 langs, e.g. ['en', 'ru'], but got {langs}. "
										    '''Usually that means that tokenizer can\'t find a mapping for the given model path '''
										    '''in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer.'''      )

							with open(UpperCAmelCase       ,	encoding='''utf-8'''      ) as src_vocab_handle:
										lowerCamelCase_ 		=     json.load(UpperCAmelCase      )
							with open(UpperCAmelCase       ,	encoding='''utf-8'''      ) as tgt_vocab_handle:
										lowerCamelCase_ 		=     json.load(UpperCAmelCase      )
										lowerCamelCase_ 		=     {v: k for k, v in tgt_vocab.items()}
							with open(UpperCAmelCase       ,	encoding='''utf-8'''      ) as merges_handle:
										lowerCamelCase_ 		=     merges_handle.read().split('''\n'''      )[:-1]
							lowerCamelCase_ 		=     [tuple(merge.split()[:2]      ) for merge in merges]
							lowerCamelCase_ 		=     dict(zip(UpperCAmelCase       ,	range(len(UpperCAmelCase      )      )      )      )
							lowerCamelCase_ 		=     {}




				def 					UpperCAmelCase__	(							self      ):
							return self.get_src_vocab()




				@property
				def 					UpperCAmelCase__	(							self      ):
							return self.src_vocab_size




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase       ,	UpperCAmelCase      ):
							if lang not in self.cache_moses_punct_normalizer:
										lowerCamelCase_ 		=     self.sm.MosesPunctNormalizer(lang=UpperCAmelCase      )
										lowerCamelCase_ 		=     punct_normalizer
							return self.cache_moses_punct_normalizer[lang].normalize(UpperCAmelCase      )




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase       ,	UpperCAmelCase      ):
							if lang not in self.cache_moses_tokenizer:
										lowerCamelCase_ 		=     self.sm.MosesTokenizer(lang=UpperCAmelCase      )
										lowerCamelCase_ 		=     moses_tokenizer
							return self.cache_moses_tokenizer[lang].tokenize(
							    UpperCAmelCase       ,	aggressive_dash_splits=UpperCAmelCase       ,	return_str=UpperCAmelCase       ,	escape=UpperCAmelCase      )




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase       ,	UpperCAmelCase      ):
							if lang not in self.cache_moses_detokenizer:
										lowerCamelCase_ 		=     self.sm.MosesDetokenizer(lang=UpperCAmelCase      )
										lowerCamelCase_ 		=     moses_detokenizer
							return self.cache_moses_detokenizer[lang].detokenize(UpperCAmelCase      )




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase       ,	UpperCAmelCase      ):
							lowerCamelCase_ 		=     replace_unicode_punct(UpperCAmelCase      )
							lowerCamelCase_ 		=     self.moses_punct_norm(UpperCAmelCase       ,	UpperCAmelCase      )
							lowerCamelCase_ 		=     remove_non_printing_char(UpperCAmelCase      )
							return text




				@property
				def 					UpperCAmelCase__	(							self      ):
							return len(self.encoder      )




				@property
				def 					UpperCAmelCase__	(							self      ):
							return len(self.decoder      )




				def 					UpperCAmelCase__	(							self      ):
							return dict(self.encoder       ,	**self.added_tokens_encoder      )




				def 					UpperCAmelCase__	(							self      ):
							return dict(self.decoder       ,	**self.added_tokens_decoder      )




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase      ):
							lowerCamelCase_ 		=     tuple(token[:-1]      ) + (token[-1] + '''</w>''',)
							if token in self.cache:
										return self.cache[token]
							lowerCamelCase_ 		=     get_pairs(UpperCAmelCase      )

							if not pairs:
										return token + "</w>"

							while True:
										lowerCamelCase_ 		=     min(UpperCAmelCase       ,	key=lambda UpperCAmelCase      : self.bpe_ranks.get(UpperCAmelCase       ,	float('''inf'''      )      )      )
										if bigram not in self.bpe_ranks:
													break
										lowerCamelCase_		,   lowerCamelCase_ 		=     bigram
										lowerCamelCase_ 		=     []
										lowerCamelCase_ 		=     0
										while i < len(UpperCAmelCase      ):
													try:
																lowerCamelCase_ 		=     word.index(UpperCAmelCase       ,	UpperCAmelCase      )
													except ValueError:
																new_word.extend(word[i:]      )
																break
													else:
																new_word.extend(word[i:j]      )
																lowerCamelCase_ 		=     j

													if word[i] == first and i < len(UpperCAmelCase      ) - 1 and word[i + 1] == second:
																new_word.append(first + second      )
																i += 2
													else:
																new_word.append(word[i]      )
																i += 1
										lowerCamelCase_ 		=     tuple(UpperCAmelCase      )
										lowerCamelCase_ 		=     new_word
										if len(UpperCAmelCase      ) == 1:
													break
										else:
													lowerCamelCase_ 		=     get_pairs(UpperCAmelCase      )
							lowerCamelCase_ 		=     ''' '''.join(UpperCAmelCase      )
							if word == "\n  </w>":
										lowerCamelCase_ 		=     '''\n</w>'''
							lowerCamelCase_ 		=     word
							return word




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase       ,	UpperCAmelCase="en"       ,	UpperCAmelCase=False      ):
							lowerCamelCase_ 		=     self.src_lang

							if self.do_lower_case:
										lowerCamelCase_ 		=     text.lower()

							if bypass_tokenizer:
										lowerCamelCase_ 		=     text.split()
							else:
										lowerCamelCase_ 		=     self.moses_pipeline(UpperCAmelCase       ,	lang=UpperCAmelCase      )
										lowerCamelCase_ 		=     self.moses_tokenize(UpperCAmelCase       ,	lang=UpperCAmelCase      )

							lowerCamelCase_ 		=     []
							for token in text:
										if token:
													split_tokens.extend(list(self.bpe(UpperCAmelCase      ).split(''' '''      )      )      )

							return split_tokens




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase      ):
							return self.encoder.get(UpperCAmelCase       ,	self.encoder.get(self.unk_token      )      )




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase      ):
							return self.decoder.get(UpperCAmelCase       ,	self.unk_token      )




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase      ):
							lowerCamelCase_ 		=     [t.replace(''' '''       ,	''''''      ).replace('''</w>'''       ,	''' '''      ) for t in tokens]
							lowerCamelCase_ 		=     ''''''.join(UpperCAmelCase      ).split()
							# detokenize
							lowerCamelCase_ 		=     self.moses_detokenize(UpperCAmelCase       ,	self.tgt_lang      )
							return text




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase       ,	UpperCAmelCase = None      ):
							lowerCamelCase_ 		=     [self.sep_token_id]

							# no bos used in fairseq
							if token_ids_a is None:
										return token_ids_a + sep
							return token_ids_a + sep + token_ids_a + sep




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase       ,	UpperCAmelCase = None       ,	UpperCAmelCase = False      ):

							if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=UpperCAmelCase       ,	token_ids_a=UpperCAmelCase       ,	already_has_special_tokens=UpperCAmelCase      )
							# no bos used in fairseq
							if token_ids_a is not None:
										return ([0] * len(UpperCAmelCase      )) + [1] + ([0] * len(UpperCAmelCase      )) + [1]
							return ([0] * len(UpperCAmelCase      )) + [1]




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase       ,	UpperCAmelCase = None      ):
							lowerCamelCase_ 		=     [self.sep_token_id]

							# no bos used in fairseq
							if token_ids_a is None:
										return len(token_ids_a + sep      ) * [0]
							return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]




				def 					UpperCAmelCase__	(							self       ,	UpperCAmelCase       ,	UpperCAmelCase = None      ):
							if not os.path.isdir(UpperCAmelCase      ):
										logger.error(f"Vocabulary path ({save_directory}) should be a directory"      )
										return

							lowerCamelCase_ 		=     os.path.join(
							    UpperCAmelCase       ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''src_vocab_file''']      )
							lowerCamelCase_ 		=     os.path.join(
							    UpperCAmelCase       ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''tgt_vocab_file''']      )
							lowerCamelCase_ 		=     os.path.join(
							    UpperCAmelCase       ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']      )

							with open(UpperCAmelCase       ,	'''w'''       ,	encoding='''utf-8'''      ) as f:
										f.write(json.dumps(self.encoder       ,	indent=2       ,	sort_keys=UpperCAmelCase       ,	ensure_ascii=UpperCAmelCase      ) + '''\n'''      )

							with open(UpperCAmelCase       ,	'''w'''       ,	encoding='''utf-8'''      ) as f:
										lowerCamelCase_ 		=     {v: k for k, v in self.decoder.items()}
										f.write(json.dumps(UpperCAmelCase       ,	indent=2       ,	sort_keys=UpperCAmelCase       ,	ensure_ascii=UpperCAmelCase      ) + '''\n'''      )

							lowerCamelCase_ 		=     0
							with open(UpperCAmelCase       ,	'''w'''       ,	encoding='''utf-8'''      ) as writer:
										for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,	key=lambda UpperCAmelCase      : kv[1]      ):
													if index != token_index:
																logger.warning(
																    f"Saving vocabulary to {merges_file}: BPE merge indices are not consecutive."
																    ''' Please check that the tokenizer is not corrupted!'''      )
																lowerCamelCase_ 		=     token_index
													writer.write(''' '''.join(UpperCAmelCase      ) + '''\n'''      )
													index += 1

							return src_vocab_file, tgt_vocab_file, merges_file




				def __getstate__(							self      ):
							lowerCamelCase_ 		=     self.__dict__.copy()
							lowerCamelCase_ 		=     None
							return state





				def __setstate__(							self       ,	UpperCAmelCase      ):
							lowerCamelCase_ 		=     d

							try:
										import sacremoses
							except ImportError:
										raise ImportError(
										    '''You need to install sacremoses to use XLMTokenizer. '''
										    '''See https://pypi.org/project/sacremoses/ for installation.'''      )

							lowerCamelCase_ 		=     sacremoses





"""simple docstring"""


import copy

from transformers.configuration_utils import PretrainedConfig
from transformers.utils import logging


A_						       =				logging.get_logger(__name__)




class    __lowerCamelCase     (   lowerCAmelCase ):
				a__:			List[str]									=						'hybrid-clip'
				a__:			List[Any]									=						True




				def __init__(							self       ,	UpperCAmelCase=512       ,	**UpperCAmelCase      ):
							super().__init__(**UpperCAmelCase      )

							if "text_config" not in kwargs:
										raise ValueError('''`text_config` can not be `None`.'''      )

							if "vision_config" not in kwargs:
										raise ValueError('''`vision_config` can not be `None`.'''      )

							lowerCamelCase_ 		=     kwargs.pop('''text_config'''      )
							lowerCamelCase_ 		=     kwargs.pop('''vision_config'''      )

							lowerCamelCase_ 		=     text_config.pop('''model_type'''      )
							lowerCamelCase_ 		=     vision_config.pop('''model_type'''      )

							from transformers import AutoConfig

							lowerCamelCase_ 		=     AutoConfig.for_model(UpperCAmelCase       ,	**UpperCAmelCase      )

							if vision_model_type == "clip":
										lowerCamelCase_ 		=     AutoConfig.for_model(UpperCAmelCase       ,	**UpperCAmelCase      ).vision_config
							elif vision_model_type == "clip_vision_model":
										from transformers import CLIPVisionConfig

										lowerCamelCase_ 		=     CLIPVisionConfig(**UpperCAmelCase      )
							else:
										lowerCamelCase_ 		=     AutoConfig.for_model(UpperCAmelCase       ,	**UpperCAmelCase      )

							lowerCamelCase_ 		=     projection_dim
							lowerCamelCase_ 		=     1.0




				@classmethod
				def 					UpperCAmelCase__	(							cls       ,	UpperCAmelCase       ,	UpperCAmelCase       ,	**UpperCAmelCase      ):
							return cls(text_config=text_config.to_dict()       ,	vision_config=vision_config.to_dict()       ,	**UpperCAmelCase      )





				def 					UpperCAmelCase__	(							self      ):
							lowerCamelCase_ 		=     copy.deepcopy(self.__dict__      )
							lowerCamelCase_ 		=     self.text_config.to_dict()
							lowerCamelCase_ 		=     self.vision_config.to_dict()
							lowerCamelCase_ 		=     self.__class__.model_type
							return output



