


'''simple docstring'''
import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 		UpperCamelCase_		(					A   ,		unittest.TestCase   ):





						"""simple docstring"""




						UpperCAmelCase__	:       Any         =     PreTrainedTokenizerFast
						UpperCAmelCase__	:       List[Any]         =     False
						UpperCAmelCase__	:       Tuple         =     True
						UpperCAmelCase__	:       Tuple         =     '''tokenizer_file'''
						def 					__A					( self				:					List[Any]  )							->					List[Any]:
										__magic_name__	  = False  # because we don't have pretrained_vocab_files_map
										super().setUp()
										__magic_name__	  = True

										__magic_name__	  = ["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
										__magic_name__	  = "SaulLu/dummy-tokenizer-bytelevel-bpe"

										# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
										__magic_name__	  = [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

										__magic_name__	  = PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
										tokenizer.save_pretrained(self.tmpdirname  )
						def 					__A					( self				:					int  )							->					Tuple:
										# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
										# model
										pass
						def 					__A					( self				:					int  )							->					str:
										# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
										# model
										pass
						def 					__A					( self				:					Union[str, Any]  )							->					Dict:
										# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
										# model
										pass
						def 					__A					( self				:					int  )							->					str:
										# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
										pass
						def 					__A					( self				:					str  )							->					int:
										__magic_name__	  = self.tmpdirname
										# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
																		try:
																						__magic_name__	  = tempfile.mkdtemp()
																						__magic_name__	  = self.rust_tokenizer_class.from_pretrained(_lowerCamelCase	,				**_lowerCamelCase  )

																						tokenizer.save_pretrained(self.tmpdirname  )
																						super().test_training_new_tokenizer()
																		finally:
																						# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																						# is restored
																						shutil.rmtree(self.tmpdirname  )
																						__magic_name__	  = tmpdirname_orig
						def 					__A					( self				:					Dict  )							->					List[str]:
										__magic_name__	  = self.tmpdirname
										# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
																		try:
																						__magic_name__	  = tempfile.mkdtemp()
																						__magic_name__	  = self.rust_tokenizer_class.from_pretrained(_lowerCamelCase	,				**_lowerCamelCase  )

																						tokenizer.save_pretrained(self.tmpdirname  )
																						super().test_training_new_tokenizer_with_special_tokens_change()
																		finally:
																						# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																						# is restored
																						shutil.rmtree(self.tmpdirname  )
																						__magic_name__	  = tmpdirname_orig

						def 					__A					( self				:					str  )							->					List[Any]:
										__magic_name__	  = self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

										__magic_name__	  = ("a" for _ in range(10_00  ))
										__magic_name__	  = tokenizer.train_new_from_iterator(text_iterator=_lowerCamelCase	,				length=10_00	,				vocab_size=50  )

										__magic_name__	  = new_tokenizer.encode("a🤗"  )
										self.assertEqual(_lowerCamelCase	,				[64, 1_72, 2_53, 97, 2_45]  )






@require_tokenizers
class 		UpperCamelCase_		(					unittest.TestCase   ):





						"""simple docstring"""




						def 					__A					( self				:					List[Any]  )							->					Optional[Any]:
										__magic_name__	  = AutoTokenizer.from_pretrained("bert-base-cased"  )
										__magic_name__	  = json.loads(tokenizer._tokenizer.to_str()  )
										__magic_name__	  = len(_lowerCamelCase  )

										with tempfile.TemporaryDirectory() as tmp_dir:
														# Hack to save this in the tokenizer_config.json
														__magic_name__	  = ["tokenizer.4.0.0.json"]
														tokenizer.save_pretrained(_lowerCamelCase  )
														json.dump(_lowerCamelCase	,				open(os.path.join(_lowerCamelCase	,				"tokenizer.4.0.0.json"  )	,				"w"  )  )

														# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
														__magic_name__	  = AutoTokenizer.from_pretrained(_lowerCamelCase  )
														self.assertEqual(len(_lowerCamelCase  )	,				len(_lowerCamelCase  ) + 1  )
														__magic_name__	  = json.loads(new_tokenizer._tokenizer.to_str()  )
														self.assertIn("huggingface"	,				json_tokenizer["model"]["vocab"]  )

														# Will need to be adjusted if we reach v42 and this test is still here.
														# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
														shutil.move(os.path.join(_lowerCamelCase	,				"tokenizer.4.0.0.json"  )	,				os.path.join(_lowerCamelCase	,				"tokenizer.42.0.0.json"  )  )
														__magic_name__	  = ["tokenizer.42.0.0.json"]
														tokenizer.save_pretrained(_lowerCamelCase  )
														__magic_name__	  = AutoTokenizer.from_pretrained(_lowerCamelCase  )
														self.assertEqual(len(_lowerCamelCase  )	,				len(_lowerCamelCase  )  )
														__magic_name__	  = json.loads(new_tokenizer._tokenizer.to_str()  )
														self.assertNotIn("huggingface"	,				json_tokenizer["model"]["vocab"]  )

						def 					__A					( self				:					Dict  )							->					List[Any]:
										# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
										__magic_name__	  = "hf-internal-testing/test-two-tokenizers"

										# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
										__magic_name__	  = AutoTokenizer.from_pretrained(_lowerCamelCase  )
										self.assertEqual(len(_lowerCamelCase  )	,				2_89_97  )
										__magic_name__	  = json.loads(tokenizer._tokenizer.to_str()  )
										self.assertIn("huggingface"	,				json_tokenizer["model"]["vocab"]  )

										# Testing an older version by monkey-patching the version in the module it's used.
										import transformers as old_transformers

										__magic_name__	  = "3.0.0"
										__magic_name__	  = old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCamelCase  )
										self.assertEqual(len(_lowerCamelCase  )	,				2_89_96  )
										__magic_name__	  = json.loads(old_tokenizer._tokenizer.to_str()  )
										self.assertNotIn("huggingface"	,				json_tokenizer["model"]["vocab"]  )






@require_tokenizers
class 		UpperCamelCase_		(					unittest.TestCase   ):





						"""simple docstring"""




						def 					__A					( self				:					int  )							->					List[Any]:
										# See https://github.com/huggingface/transformers/pull/12550
										# and https://github.com/huggingface/tokenizers/issues/537
										__magic_name__	  = PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"  )
										__magic_name__	  = "The Matrix is a 1999 science fiction action film."

										with concurrent.futures.ThreadPoolExecutor() as executor:
														__magic_name__	  = [executor.submit(self.fetch	,				_lowerCamelCase	,				_lowerCamelCase  ) for i in range(10  )]
														__magic_name__	  = [future.result() for future in futures]
														self.assertEqual(_lowerCamelCase	,				[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )

						def 					__A					( self				:					List[Any]	,				_lowerCamelCase				:					Optional[Any]	,				_lowerCamelCase				:					Dict  )							->					int:
										return tokenizer.encode(_lowerCamelCase	,				truncation="longest_first"	,				padding="longest"  )







'''simple docstring'''
import os
import sys
import time

from . import transposition_cipher as trans_cipher




def      __snake_case       (       ):
				'''simple docstring'''




				__magic_name__	  = "Prehistoric Men.txt"
				__magic_name__	  = "Output.txt"
				__magic_name__	  = int(input("Enter key: "						)						)
				__magic_name__	  = input("Encrypt/Decrypt [e/d]: "						)

				if not os.path.exists(lowerCamelCase_						):
								print(F'File {input_file} does not exist. Quitting...'						)
								sys.exit()
				if os.path.exists(lowerCamelCase_						):
								print(F'Overwrite {output_file}? [y/n]'						)
								__magic_name__	  = input("> "						)
								if not response.lower().startswith("y"						):
												sys.exit()

				__magic_name__	  = time.time()
				if mode.lower().startswith("e"						):
								with open(lowerCamelCase_						) as f:
												__magic_name__	  = f.read()
								__magic_name__	  = trans_cipher.encrypt_message(lowerCamelCase_	,  lowerCamelCase_						)
				elif mode.lower().startswith("d"						):
								with open(lowerCamelCase_						) as f:
												__magic_name__	  = f.read()
								__magic_name__	  = trans_cipher.decrypt_message(lowerCamelCase_	,  lowerCamelCase_						)

				with open(lowerCamelCase_	,  "w"						) as output_obj:
								output_obj.write(lowerCamelCase_						)

				__magic_name__	  = round(time.time() - start_time	,  2						)
				print(("Done (", total_time, "seconds )")						)


if __name__ == "__main__":
						main()



