


from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

UpperCamelCase :				List[str]						    =       {
    name: (getattr(transformers, name), getattr(transformers, name + '''Fast''')) for name in SLOW_TO_FAST_CONVERTERS
}

UpperCamelCase :				Any						    =       datasets.load_dataset('''xnli''', split='''test+validation''')

UpperCamelCase :				str						    =       0
UpperCamelCase :				Union[str, Any]						    =       0
UpperCamelCase :				Any						    =       0
UpperCamelCase :				Union[str, Any]						    =       0
def 							UpperCamelCase_  (				__snake_case			:						List[Any]	,			__snake_case			:						List[Any]	,			__snake_case			:						List[Any]	,			__snake_case			:						Union[str, Any]     )->      Tuple:




						'''simple docstring'''
						if spm_diff == list(reversed(__snake_case     )     ):
												# AAA -> AA+A vs A+AA case.
												return True
						elif len(__snake_case     ) == len(__snake_case     ) and fast.decode(__snake_case     ) == fast.decode(__snake_case     ):
												# Second order OK
												# Barrich -> Barr + ich vs Bar + rich
												return True
						__magic_name__ 		=			slow.encode(slow.decode(__snake_case     )     )
						__magic_name__ 		=			fast.encode(fast.decode(__snake_case     )     )
						if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
												# Type 3 error.
												# Snehagatha ->
												#       Sne, h, aga, th, a
												#       Sne, ha, gat, ha
												# Encoding the wrong with sp does not even recover what spm gave us
												# It fits tokenizer however...
												return True
						return False
def 							UpperCamelCase_  (				__snake_case			:						Tuple	,			__snake_case			:						Union[str, Any]	,			__snake_case			:						Tuple     )->      Any:




						'''simple docstring'''
						__magic_name__ 		=			fast.encode_plus(__snake_case     )[0]
						__magic_name__ 		=			enc.offsets
						__magic_name__		,			__magic_name__ 		=			offsets[idx], offsets[idx - 1]
						if curr is not None and line[curr[0] : curr[1]] == "\u200f":
												return True
						if prev is not None and line[prev[0] : prev[1]] == "\u200f":
												return True
def 							UpperCamelCase_  (				__snake_case			:						Optional[int]	,			__snake_case			:						Optional[int]	,			__snake_case			:						Optional[Any]	,			__snake_case			:						List[str]	,			__snake_case			:						List[Any]     )->      Union[str, Any]:




						'''simple docstring'''
						# Encoding can be the same with same result AAA -> A + AA vs AA + A
						# We can check that we use at least exactly the same number of tokens.
						for i, (spm_id, tok_id) in enumerate(zip(__snake_case	,			__snake_case     )     ):
												if spm_id != tok_id:
																		break
						__magic_name__ 		=			i
						for i, (spm_id, tok_id) in enumerate(zip(reversed(__snake_case     )	,			reversed(__snake_case     )     )     ):
												if spm_id != tok_id:
																		break
						__magic_name__ 		=			len(__snake_case     ) - i

						__magic_name__ 		=			spm_ids[first:last]
						__magic_name__ 		=			tok_ids[first:last]

						if check_diff(__snake_case	,			__snake_case	,			__snake_case	,			__snake_case     ):
												return True

						if check_LTR_mark(__snake_case	,			__snake_case	,			__snake_case     ):
												return True

						if last - first > 5:
												# We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
												__magic_name__ 		=			Counter(spm_ids[first:last]     )
												__magic_name__ 		=			Counter(tok_ids[first:last]     )

												__magic_name__ 		=			{spm_ for (spm_, si) in spms.items() if toks.get(spm_	,			0     ) == si}
												__magic_name__ 		=			3
												for i in range(last - first - min_width     ):
																		if all(spm_ids[first + i + j] in removable_tokens for j in range(__snake_case     )     ):
																								__magic_name__ 		=			[
																								    k
																								    for k in range(last - first - min_width     )
																								    if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
																								]
																								for j in possible_matches:
																														if check_diff(spm_ids[first : first + i]	,			tok_ids[first : first + j]	,			__snake_case	,			__snake_case     ) and check_details(
																														    __snake_case	,			spm_ids[first + i : last]	,			tok_ids[first + j : last]	,			__snake_case	,			__snake_case	,			):
																																				return True

						print(F'''Spm: {[fast.decode([spm_ids[i]]     ) for i in range(__snake_case	,			__snake_case     )]}'''     )
						try:
												print(F'''Tok: {[fast.decode([tok_ids[i]]     ) for i in range(__snake_case	,			__snake_case     )]}'''     )
						except Exception:
												pass

						__magic_name__ 		=			fast.decode(spm_ids[:first]     )
						__magic_name__ 		=			fast.decode(spm_ids[last:]     )
						__magic_name__ 		=			fast.decode(spm_ids[first:last]     )
						print()
						print(__snake_case     )
						return False
def 							UpperCamelCase_  (				__snake_case			:						int	,			__snake_case			:						Optional[int]	,			__snake_case			:						Dict     )->      Optional[Any]:




						'''simple docstring'''
						global perfect
						global imperfect
						global wrong
						global total

						__magic_name__ 		=			slow.encode(__snake_case     )
						__magic_name__ 		=			fast.encode(__snake_case     )

						__magic_name__ 		=			False
						total += 1

						if slow_ids != fast_ids:
												if check_details(__snake_case	,			__snake_case	,			__snake_case	,			__snake_case	,			__snake_case     ):
																		__magic_name__ 		=			True
																		imperfect += 1
												else:
																		wrong += 1
						else:
												perfect += 1

						if total % 1_00_00 == 0:
												print(F'''({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})'''     )

						if skip_assert:
												return

						assert (
						    slow_ids == fast_ids
						), F'''line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(__snake_case     )}\n{fast.tokenize(__snake_case     )}'''
def 							UpperCamelCase_  (				__snake_case			:						Tuple	,			__snake_case			:						int     )->      List[Any]:




						'''simple docstring'''
						global batch_total
						for i in range(len(__snake_case     )     ):
												# premise, all languages
												for text in dataset[i]["premise"].values():
																		test_string(__snake_case	,			__snake_case	,			__snake_case     )

												# hypothesis, all languages
												for text in dataset[i]["hypothesis"]["translation"]:
																		test_string(__snake_case	,			__snake_case	,			__snake_case     )


if __name__ == "__main__":
					for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
										UpperCamelCase :				str						    =       list(slow_class.max_model_input_sizes.keys())
										for checkpoint in checkpoint_names:
															UpperCamelCase :				Any						    =       0
															UpperCamelCase :				Any						    =       0
															UpperCamelCase :				Optional[int]						    =       0
															UpperCamelCase :				Dict						    =       0

															print(f'''========================== Checking {name}: {checkpoint} ==========================''')
															UpperCamelCase :				Tuple						    =       slow_class.from_pretrained(checkpoint, force_download=True)
															UpperCamelCase :				Optional[Any]						    =       fast_class.from_pretrained(checkpoint, force_download=True)
															test_tokenizer(slow, fast)
															print(f'''Accuracy {perfect * 100 / total:.2f}''')





def 							UpperCamelCase_  (				__snake_case			:						str     )->      int:




						'''simple docstring'''
						__magic_name__ 		=			__magic_name__ 		=			0
						for letter in data:
												__magic_name__ 		=			(hash_ << 4) + ord(__snake_case     )
												__magic_name__ 		=			hash_ & 0xf000_0000
												if x != 0:
																		hash_ ^= x >> 24
												hash_ &= ~x
						return hash_


if __name__ == "__main__":
					import doctest

					doctest.testmod()

