




import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase_			:       Any						       =logging.get_logger(__name__)

lowercase_			:       str						       ={
    'src_vocab_file': 'vocab-src.json',
    'tgt_vocab_file': 'vocab-tgt.json',
    'merges_file': 'merges.txt',
}

lowercase_			:       str						       ={
    'src_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json'
    },
    'tgt_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json'
    },
    'merges_file': {'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt'},
}

lowercase_			:       Dict						       ={'stas/tiny-wmt19-en-de': 10_24}
lowercase_			:       Optional[int]						       ={
    'stas/tiny-wmt19-en-de': {
        'langs': ['en', 'de'],
        'model_max_length': 10_24,
        'special_tokens_map_file': None,
        'full_tokenizer_file': None,
    }
}


def 							__SCREAMING_SNAKE_CASE						(    snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase									=  set()
						_UpperCAmelCase									=  word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)		)
												_UpperCAmelCase									=  char
						return pairs


def 							__SCREAMING_SNAKE_CASE						(    snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase									=  text.replace("，"			,					","		)
						_UpperCAmelCase									=  re.sub(R"。\s*"			,					". "			,					snake_case_		)
						_UpperCAmelCase									=  text.replace("、"			,					","		)
						_UpperCAmelCase									=  text.replace("”"			,					"\""		)
						_UpperCAmelCase									=  text.replace("“"			,					"\""		)
						_UpperCAmelCase									=  text.replace("∶"			,					":"		)
						_UpperCAmelCase									=  text.replace("："			,					":"		)
						_UpperCAmelCase									=  text.replace("？"			,					"?"		)
						_UpperCAmelCase									=  text.replace("《"			,					"\""		)
						_UpperCAmelCase									=  text.replace("》"			,					"\""		)
						_UpperCAmelCase									=  text.replace("）"			,					")"		)
						_UpperCAmelCase									=  text.replace("！"			,					"!"		)
						_UpperCAmelCase									=  text.replace("（"			,					"("		)
						_UpperCAmelCase									=  text.replace("；"			,					";"		)
						_UpperCAmelCase									=  text.replace("１"			,					"1"		)
						_UpperCAmelCase									=  text.replace("」"			,					"\""		)
						_UpperCAmelCase									=  text.replace("「"			,					"\""		)
						_UpperCAmelCase									=  text.replace("０"			,					"0"		)
						_UpperCAmelCase									=  text.replace("３"			,					"3"		)
						_UpperCAmelCase									=  text.replace("２"			,					"2"		)
						_UpperCAmelCase									=  text.replace("５"			,					"5"		)
						_UpperCAmelCase									=  text.replace("６"			,					"6"		)
						_UpperCAmelCase									=  text.replace("９"			,					"9"		)
						_UpperCAmelCase									=  text.replace("７"			,					"7"		)
						_UpperCAmelCase									=  text.replace("８"			,					"8"		)
						_UpperCAmelCase									=  text.replace("４"			,					"4"		)
						_UpperCAmelCase									=  re.sub(R"．\s*"			,					". "			,					snake_case_		)
						_UpperCAmelCase									=  text.replace("～"			,					"~"		)
						_UpperCAmelCase									=  text.replace("’"			,					"'"		)
						_UpperCAmelCase									=  text.replace("…"			,					"..."		)
						_UpperCAmelCase									=  text.replace("━"			,					"-"		)
						_UpperCAmelCase									=  text.replace("〈"			,					"<"		)
						_UpperCAmelCase									=  text.replace("〉"			,					">"		)
						_UpperCAmelCase									=  text.replace("【"			,					"["		)
						_UpperCAmelCase									=  text.replace("】"			,					"]"		)
						_UpperCAmelCase									=  text.replace("％"			,					"%"		)
						return text


def 							__SCREAMING_SNAKE_CASE						(    snake_case_		):

						'''simple docstring'''


						_UpperCAmelCase									=  []
						for char in text:
												_UpperCAmelCase									=  unicodedata.category(snake_case_		)
												if cat.startswith("C"		):
																		continue
												output.append(snake_case_		)
						return "".join(snake_case_		)






class 					__lowerCAmelCase				( UpperCAmelCase__				):
			snake_case_   :				Optional[Any]				  =  VOCAB_FILES_NAMES
			snake_case_   :				Tuple				  =  PRETRAINED_VOCAB_FILES_MAP
			snake_case_   :				List[Any]				  =  PRETRAINED_INIT_CONFIGURATION
			snake_case_   :				Any				  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case_   :				List[str]				  =  ["input_ids", "attention_mask"]







			def __init__( self					:    Union[str, Any]					,					snake_case__					:    int=None					,					snake_case__					:    Optional[Any]=None					,					snake_case__					:    Tuple=None					,					snake_case__					:    Optional[int]=None					,					snake_case__					:    Dict=False					,					snake_case__					:    Tuple="<unk>"					,					snake_case__					:    Tuple="<s>"					,					snake_case__					:    List[Any]="</s>"					,					snake_case__					:    Union[str, Any]="<pad>"					,					**snake_case__					:    str					,					):




									"""simple docstring"""


									super().__init__(
									    langs=snake_case__					,					src_vocab_file=snake_case__					,					tgt_vocab_file=snake_case__					,					merges_file=snake_case__					,					do_lower_case=snake_case__					,					unk_token=snake_case__					,					bos_token=snake_case__					,					sep_token=snake_case__					,					pad_token=snake_case__					,					**snake_case__					,					)

									try:
															import sacremoses
									except ImportError:
															raise ImportError(
															    "You need to install sacremoses to use XLMTokenizer. "
															    "See https://pypi.org/project/sacremoses/ for installation."   )

									_UpperCAmelCase									=  sacremoses

									_UpperCAmelCase									=  src_vocab_file
									_UpperCAmelCase									=  tgt_vocab_file
									_UpperCAmelCase									=  merges_file
									_UpperCAmelCase									=  do_lower_case

									# cache of sm.MosesPunctNormalizer instance
									_UpperCAmelCase									=  {}
									# cache of sm.MosesTokenizer instance
									_UpperCAmelCase									=  {}
									_UpperCAmelCase									=  {}

									if langs and len(snake_case__   ) == 2:
															_UpperCAmelCase							,				_UpperCAmelCase									=  langs
									else:
															raise ValueError(
															    F"""arg `langs` needs to be a list of 2 langs, e.g. ['en', 'ru'], but got {langs}. """
															    "Usually that means that tokenizer can't find a mapping for the given model path "
															    "in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer."   )

									with open(snake_case__					,					encoding="utf-8"   ) as src_vocab_handle:
															_UpperCAmelCase									=  json.load(snake_case__   )
									with open(snake_case__					,					encoding="utf-8"   ) as tgt_vocab_handle:
															_UpperCAmelCase									=  json.load(snake_case__   )
															_UpperCAmelCase									=  {v: k for k, v in tgt_vocab.items()}
									with open(snake_case__					,					encoding="utf-8"   ) as merges_handle:
															_UpperCAmelCase									=  merges_handle.read().split("\n"   )[:-1]
									_UpperCAmelCase									=  [tuple(merge.split()[:2]   ) for merge in merges]
									_UpperCAmelCase									=  dict(zip(snake_case__					,					range(len(snake_case__   )   )   )   )
									_UpperCAmelCase									=  {}







			def UpperCamelCase    ( self					:    Union[str, Any]   ):




									"""simple docstring"""


									return self.get_src_vocab()







			@property
			def UpperCamelCase    ( self					:    str   ):




									"""simple docstring"""


									return self.src_vocab_size







			def UpperCamelCase    ( self					:    List[Any]					,					snake_case__					:    int					,					snake_case__					:    List[str]   ):




									"""simple docstring"""


									if lang not in self.cache_moses_punct_normalizer:
															_UpperCAmelCase									=  self.sm.MosesPunctNormalizer(lang=snake_case__   )
															_UpperCAmelCase									=  punct_normalizer
									return self.cache_moses_punct_normalizer[lang].normalize(snake_case__   )







			def UpperCamelCase    ( self					:    List[str]					,					snake_case__					:    Union[str, Any]					,					snake_case__					:    Tuple   ):




									"""simple docstring"""


									if lang not in self.cache_moses_tokenizer:
															_UpperCAmelCase									=  self.sm.MosesTokenizer(lang=snake_case__   )
															_UpperCAmelCase									=  moses_tokenizer
									return self.cache_moses_tokenizer[lang].tokenize(
									    snake_case__					,					aggressive_dash_splits=snake_case__					,					return_str=snake_case__					,					escape=snake_case__   )







			def UpperCamelCase    ( self					:    Any					,					snake_case__					:    str					,					snake_case__					:    Optional[Any]   ):




									"""simple docstring"""


									if lang not in self.cache_moses_detokenizer:
															_UpperCAmelCase									=  self.sm.MosesDetokenizer(lang=snake_case__   )
															_UpperCAmelCase									=  moses_detokenizer
									return self.cache_moses_detokenizer[lang].detokenize(snake_case__   )







			def UpperCamelCase    ( self					:    Union[str, Any]					,					snake_case__					:    Any					,					snake_case__					:    Union[str, Any]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  replace_unicode_punct(snake_case__   )
									_UpperCAmelCase									=  self.moses_punct_norm(snake_case__					,					snake_case__   )
									_UpperCAmelCase									=  remove_non_printing_char(snake_case__   )
									return text







			@property
			def UpperCamelCase    ( self					:    Optional[int]   ):




									"""simple docstring"""


									return len(self.encoder   )







			@property
			def UpperCamelCase    ( self					:    Union[str, Any]   ):




									"""simple docstring"""


									return len(self.decoder   )







			def UpperCamelCase    ( self					:    Union[str, Any]   ):




									"""simple docstring"""


									return dict(self.encoder					,					**self.added_tokens_encoder   )







			def UpperCamelCase    ( self					:    Optional[Any]   ):




									"""simple docstring"""


									return dict(self.decoder					,					**self.added_tokens_decoder   )







			def UpperCamelCase    ( self					:    List[Any]					,					snake_case__					:    Any   ):




									"""simple docstring"""


									_UpperCAmelCase									=  tuple(token[:-1]   ) + (token[-1] + "</w>",)
									if token in self.cache:
															return self.cache[token]
									_UpperCAmelCase									=  get_pairs(snake_case__   )

									if not pairs:
															return token + "</w>"

									while True:
															_UpperCAmelCase									=  min(snake_case__					,					key=lambda snake_case__   : self.bpe_ranks.get(snake_case__					,					float("inf"   )   )   )
															if bigram not in self.bpe_ranks:
																					break
															_UpperCAmelCase							,				_UpperCAmelCase									=  bigram
															_UpperCAmelCase									=  []
															_UpperCAmelCase									=  0
															while i < len(snake_case__   ):
																					try:
																											_UpperCAmelCase									=  word.index(snake_case__					,					snake_case__   )
																					except ValueError:
																											new_word.extend(word[i:]   )
																											break
																					else:
																											new_word.extend(word[i:j]   )
																											_UpperCAmelCase									=  j

																					if word[i] == first and i < len(snake_case__   ) - 1 and word[i + 1] == second:
																											new_word.append(first + second   )
																											i += 2
																					else:
																											new_word.append(word[i]   )
																											i += 1
															_UpperCAmelCase									=  tuple(snake_case__   )
															_UpperCAmelCase									=  new_word
															if len(snake_case__   ) == 1:
																					break
															else:
																					_UpperCAmelCase									=  get_pairs(snake_case__   )
									_UpperCAmelCase									=  " ".join(snake_case__   )
									if word == "\n  </w>":
															_UpperCAmelCase									=  "\n</w>"
									_UpperCAmelCase									=  word
									return word







			def UpperCamelCase    ( self					:    Union[str, Any]					,					snake_case__					:    List[str]					,					snake_case__					:    Tuple="en"					,					snake_case__					:    Optional[Any]=False   ):




									"""simple docstring"""


									_UpperCAmelCase									=  self.src_lang

									if self.do_lower_case:
															_UpperCAmelCase									=  text.lower()

									if bypass_tokenizer:
															_UpperCAmelCase									=  text.split()
									else:
															_UpperCAmelCase									=  self.moses_pipeline(snake_case__					,					lang=snake_case__   )
															_UpperCAmelCase									=  self.moses_tokenize(snake_case__					,					lang=snake_case__   )

									_UpperCAmelCase									=  []
									for token in text:
															if token:
																					split_tokens.extend(list(self.bpe(snake_case__   ).split(" "   )   )   )

									return split_tokens







			def UpperCamelCase    ( self					:    Union[str, Any]					,					snake_case__					:    Optional[Any]   ):




									"""simple docstring"""


									return self.encoder.get(snake_case__					,					self.encoder.get(self.unk_token   )   )







			def UpperCamelCase    ( self					:    Union[str, Any]					,					snake_case__					:    Union[str, Any]   ):




									"""simple docstring"""


									return self.decoder.get(snake_case__					,					self.unk_token   )







			def UpperCamelCase    ( self					:    int					,					snake_case__					:    Union[str, Any]   ):




									"""simple docstring"""


									_UpperCAmelCase									=  [t.replace(" "					,					""   ).replace("</w>"					,					" "   ) for t in tokens]
									_UpperCAmelCase									=  "".join(snake_case__   ).split()
									# detokenize
									_UpperCAmelCase									=  self.moses_detokenize(snake_case__					,					self.tgt_lang   )
									return text







			def UpperCamelCase    ( self					:    Dict					,					snake_case__					:    List[int]					,					snake_case__					:    Optional[List[int]] = None   ):




									"""simple docstring"""


									_UpperCAmelCase									=  [self.sep_token_id]

									# no bos used in fairseq
									if token_ids_a is None:
															return token_ids_a + sep
									return token_ids_a + sep + token_ids_a + sep







			def UpperCamelCase    ( self					:    List[Any]					,					snake_case__					:    List[int]					,					snake_case__					:    Optional[List[int]] = None					,					snake_case__					:    bool = False   ):




									"""simple docstring"""


									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=snake_case__					,					token_ids_a=snake_case__					,					already_has_special_tokens=snake_case__   )
									# no bos used in fairseq
									if token_ids_a is not None:
															return ([0] * len(snake_case__   )) + [1] + ([0] * len(snake_case__   )) + [1]
									return ([0] * len(snake_case__   )) + [1]







			def UpperCamelCase    ( self					:    Optional[int]					,					snake_case__					:    List[int]					,					snake_case__					:    Optional[List[int]] = None   ):




									"""simple docstring"""


									_UpperCAmelCase									=  [self.sep_token_id]

									# no bos used in fairseq
									if token_ids_a is None:
															return len(token_ids_a + sep   ) * [0]
									return len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]







			def UpperCamelCase    ( self					:    Tuple					,					snake_case__					:    str					,					snake_case__					:    Optional[str] = None   ):




									"""simple docstring"""


									if not os.path.isdir(snake_case__   ):
															logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
															return

									_UpperCAmelCase									=  os.path.join(
									    snake_case__					,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["src_vocab_file"]   )
									_UpperCAmelCase									=  os.path.join(
									    snake_case__					,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["tgt_vocab_file"]   )
									_UpperCAmelCase									=  os.path.join(
									    snake_case__					,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]   )

									with open(snake_case__					,					"w"					,					encoding="utf-8"   ) as f:
															f.write(json.dumps(self.encoder					,					indent=2					,					sort_keys=snake_case__					,					ensure_ascii=snake_case__   ) + "\n"   )

									with open(snake_case__					,					"w"					,					encoding="utf-8"   ) as f:
															_UpperCAmelCase									=  {v: k for k, v in self.decoder.items()}
															f.write(json.dumps(snake_case__					,					indent=2					,					sort_keys=snake_case__					,					ensure_ascii=snake_case__   ) + "\n"   )

									_UpperCAmelCase									=  0
									with open(snake_case__					,					"w"					,					encoding="utf-8"   ) as writer:
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,					key=lambda snake_case__   : kv[1]   ):
																					if index != token_index:
																											logger.warning(
																											    F"""Saving vocabulary to {merges_file}: BPE merge indices are not consecutive."""
																											    " Please check that the tokenizer is not corrupted!"   )
																											_UpperCAmelCase									=  token_index
																					writer.write(" ".join(snake_case__   ) + "\n"   )
																					index += 1

									return src_vocab_file, tgt_vocab_file, merges_file







			def __getstate__( self					:    Dict   ):




									"""simple docstring"""


									_UpperCAmelCase									=  self.__dict__.copy()
									_UpperCAmelCase									=  None
									return state







			def __setstate__( self					:    int					,					snake_case__					:    str   ):




									"""simple docstring"""


									_UpperCAmelCase									=  d

									try:
															import sacremoses
									except ImportError:
															raise ImportError(
															    "You need to install sacremoses to use XLMTokenizer. "
															    "See https://pypi.org/project/sacremoses/ for installation."   )

									_UpperCAmelCase									=  sacremoses










import os
import shutil
import unittest
from unittest.mock import patch

from transformers.testing_utils import CaptureStd, is_pt_tf_cross_test, require_torch






class 					__lowerCAmelCase				( unittest.TestCase				):







			@patch("sys.argv"					,					["fakeprogrampath", "env"]   )
			def UpperCamelCase    ( self					:    List[str]   ):




									"""simple docstring"""


									import transformers.commands.transformers_cli

									with CaptureStd() as cs:
															transformers.commands.transformers_cli.main()
									self.assertIn("Python version"					,					cs.out   )
									self.assertIn("Platform"					,					cs.out   )
									self.assertIn("Using distributed or parallel set-up in script?"					,					cs.out   )







			@is_pt_tf_cross_test
			@patch(
			    "sys.argv"					,					["fakeprogrampath", "pt-to-tf", "--model-name", "hf-internal-testing/tiny-random-gptj", "--no-pr"]   )
			def UpperCamelCase    ( self					:    Optional[Any]   ):




									"""simple docstring"""


									import transformers.commands.transformers_cli

									shutil.rmtree("/tmp/hf-internal-testing/tiny-random-gptj"					,					ignore_errors=snake_case__   )  # cleans potential past runs
									transformers.commands.transformers_cli.main()

									# The original repo has no TF weights -- if they exist, they were created by the CLI
									self.assertTrue(os.path.exists("/tmp/hf-internal-testing/tiny-random-gptj/tf_model.h5"   )   )







			@require_torch
			@patch("sys.argv"					,					["fakeprogrampath", "download", "hf-internal-testing/tiny-random-gptj", "--cache-dir", "/tmp"]   )
			def UpperCamelCase    ( self					:    Any   ):




									"""simple docstring"""


									import transformers.commands.transformers_cli

									# # remove any previously downloaded model to start clean
									shutil.rmtree("/tmp/models--hf-internal-testing--tiny-random-gptj"					,					ignore_errors=snake_case__   )

									# run the command
									transformers.commands.transformers_cli.main()

									# check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--tiny-random-gptj
									self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/blobs"   )   )
									self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/refs"   )   )
									self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/snapshots"   )   )







			@require_torch
			@patch(
			    "sys.argv"					,					[
			        "fakeprogrampath",
			        "download",
			        "hf-internal-testing/test_dynamic_model_with_tokenizer",
			        "--trust-remote-code",
			        "--cache-dir",
			        "/tmp",
			    ]					,					)
			def UpperCamelCase    ( self					:    int   ):




									"""simple docstring"""


									import transformers.commands.transformers_cli

									# # remove any previously downloaded model to start clean
									shutil.rmtree("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer"					,					ignore_errors=snake_case__   )

									# run the command
									transformers.commands.transformers_cli.main()

									# check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer
									self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/blobs"   )   )
									self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/refs"   )   )
									self.assertTrue(
									    os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/snapshots"   )   )




