



"""simple docstring"""

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
							A_	:						int				=			None


A_	:						Dict				=			logging.get_logger(__name__)

A_	:						str				=			{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

A_	:						str				=			{
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'
        ),
    },
}

A_	:						List[Any]				=			{
    'xlm-roberta-base': 512,
    'xlm-roberta-large': 512,
    'xlm-roberta-large-finetuned-conll02-dutch': 512,
    'xlm-roberta-large-finetuned-conll02-spanish': 512,
    'xlm-roberta-large-finetuned-conll03-english': 512,
    'xlm-roberta-large-finetuned-conll03-german': 512,
}
class 						lowerCAmelCase__	(  _lowerCamelCase   ):


	'''simple docstring'''







	_SCREAMING_SNAKE_CASE  :		Optional[int]     	=   VOCAB_FILES_NAMES
	_SCREAMING_SNAKE_CASE  :		Dict     	=   PRETRAINED_VOCAB_FILES_MAP
	_SCREAMING_SNAKE_CASE  :		List[str]     	=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_SCREAMING_SNAKE_CASE  :		List[str]     	=   ['''input_ids''', '''attention_mask''']
	_SCREAMING_SNAKE_CASE  :		List[Any]     	=   XLMRobertaTokenizer





	def __init__(			self							:					Tuple    ,					_SCREAMING_SNAKE_CASE							:					str=None    ,					_SCREAMING_SNAKE_CASE							:					Tuple=None    ,					_SCREAMING_SNAKE_CASE							:					int="<s>"    ,					_SCREAMING_SNAKE_CASE							:					Union[str, Any]="</s>"    ,					_SCREAMING_SNAKE_CASE							:					Tuple="</s>"    ,					_SCREAMING_SNAKE_CASE							:					Optional[int]="<s>"    ,					_SCREAMING_SNAKE_CASE							:					str="<unk>"    ,					_SCREAMING_SNAKE_CASE							:					Optional[Any]="<pad>"    ,					_SCREAMING_SNAKE_CASE							:					Optional[int]="<mask>"    ,					**_SCREAMING_SNAKE_CASE							:					Optional[int]    ,					)	->  Union[str, Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       str 							=    AddedToken(_SCREAMING_SNAKE_CASE    ,					lstrip=_SCREAMING_SNAKE_CASE    ,					rstrip=_SCREAMING_SNAKE_CASE						) if isinstance(_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE						) else mask_token

					super().__init__(
					    _SCREAMING_SNAKE_CASE    ,					tokenizer_file=_SCREAMING_SNAKE_CASE    ,					bos_token=_SCREAMING_SNAKE_CASE    ,					eos_token=_SCREAMING_SNAKE_CASE    ,					sep_token=_SCREAMING_SNAKE_CASE    ,					cls_token=_SCREAMING_SNAKE_CASE    ,					unk_token=_SCREAMING_SNAKE_CASE    ,					pad_token=_SCREAMING_SNAKE_CASE    ,					mask_token=_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE    ,					)

					SCREAMING_SNAKE_CASE	:       List[str] 							=    vocab_file
					SCREAMING_SNAKE_CASE	:       Dict 							=    False if not self.vocab_file else True





	def  _lowerCAmelCase    (			self							:					Union[str, Any]    ,					_SCREAMING_SNAKE_CASE							:					List[int]    ,					_SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None						)	->  List[int]:





					"""simple docstring"""


					if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					SCREAMING_SNAKE_CASE	:       str 							=    [self.cls_token_id]
					SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    [self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep





	def  _lowerCAmelCase    (			self							:					List[Any]    ,					_SCREAMING_SNAKE_CASE							:					List[int]    ,					_SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None						)	->  List[int]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       str 							=    [self.sep_token_id]
					SCREAMING_SNAKE_CASE	:       List[str] 							=    [self.cls_token_id]

					if token_ids_a is None:
									return len(cls + token_ids_a + sep						) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]







	def  _lowerCAmelCase    (			self							:					List[str]    ,					_SCREAMING_SNAKE_CASE							:					str    ,					_SCREAMING_SNAKE_CASE							:					Optional[str] = None						)	->  Tuple[str]:





					"""simple docstring"""


					if not self.can_save_slow_tokenizer:
									raise ValueError(
									    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
									    'tokenizer.'						)

					if not os.path.isdir(_SCREAMING_SNAKE_CASE						):
									logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""						)
									return
					SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    os.path.join(
					    _SCREAMING_SNAKE_CASE    ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

					if os.path.abspath(self.vocab_file						) != os.path.abspath(_SCREAMING_SNAKE_CASE						):
									copyfile(self.vocab_file    ,					_SCREAMING_SNAKE_CASE						)

					return (out_vocab_file,)










"""simple docstring"""

import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
							import torch

if is_vision_available():
							from PIL import Image

							from transformers import ConvNextImageProcessor
class 						lowerCAmelCase__	(  unittest.TestCase   ):


	'''simple docstring'''







	def __init__(			self							:					Tuple    ,					_SCREAMING_SNAKE_CASE							:					List[str]    ,					_SCREAMING_SNAKE_CASE							:					List[Any]=7    ,					_SCREAMING_SNAKE_CASE							:					Optional[int]=3    ,					_SCREAMING_SNAKE_CASE							:					List[str]=18    ,					_SCREAMING_SNAKE_CASE							:					str=30    ,					_SCREAMING_SNAKE_CASE							:					str=400    ,					_SCREAMING_SNAKE_CASE							:					Any=True    ,					_SCREAMING_SNAKE_CASE							:					int=None    ,					_SCREAMING_SNAKE_CASE							:					Dict=0.8_7_5    ,					_SCREAMING_SNAKE_CASE							:					List[str]=True    ,					_SCREAMING_SNAKE_CASE							:					List[Any]=[0.5, 0.5, 0.5]    ,					_SCREAMING_SNAKE_CASE							:					str=[0.5, 0.5, 0.5]    ,					)	->  int:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       int 							=    size if size is not None else {'shortest_edge': 20}
					SCREAMING_SNAKE_CASE	:       List[Any] 							=    parent
					SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    batch_size
					SCREAMING_SNAKE_CASE	:       Any 							=    num_channels
					SCREAMING_SNAKE_CASE	:       Any 							=    image_size
					SCREAMING_SNAKE_CASE	:       str 							=    min_resolution
					SCREAMING_SNAKE_CASE	:       Dict 							=    max_resolution
					SCREAMING_SNAKE_CASE	:       Dict 							=    do_resize
					SCREAMING_SNAKE_CASE	:       List[str] 							=    size
					SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    crop_pct
					SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    do_normalize
					SCREAMING_SNAKE_CASE	:       Dict 							=    image_mean
					SCREAMING_SNAKE_CASE	:       str 							=    image_std







	def  _lowerCAmelCase    (			self							:					Dict						)	->  Any:





					"""simple docstring"""


					return {
					    "image_mean": self.image_mean,
					    "image_std": self.image_std,
					    "do_normalize": self.do_normalize,
					    "do_resize": self.do_resize,
					    "size": self.size,
					    "crop_pct": self.crop_pct,
					}




@require_torch
@require_vision
class 						lowerCAmelCase__	(  _lowerCamelCase       ,       unittest.TestCase   ):


	'''simple docstring'''







	_SCREAMING_SNAKE_CASE  :		Optional[Any]     	=   ConvNextImageProcessor if is_vision_available() else None





	def  _lowerCAmelCase    (			self							:					Any						)	->  Dict:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    ConvNextImageProcessingTester(self						)





	@property
	def  _lowerCAmelCase    (			self							:					Optional[int]						)	->  Dict:





					"""simple docstring"""


					return self.image_processor_tester.prepare_image_processor_dict()





	def  _lowerCAmelCase    (			self							:					str						)	->  int:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Any 							=    self.image_processing_class(**self.image_processor_dict						)
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE    ,					'do_resize'						)						)
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE    ,					'size'						)						)
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE    ,					'crop_pct'						)						)
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE    ,					'do_normalize'						)						)
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE    ,					'image_mean'						)						)
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE    ,					'image_std'						)						)





	def  _lowerCAmelCase    (			self							:					Union[str, Any]						)	->  Union[str, Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    self.image_processing_class.from_dict(self.image_processor_dict						)
					self.assertEqual(image_processor.size    ,					{'shortest_edge': 20}						)

					SCREAMING_SNAKE_CASE	:       Optional[int] 							=    self.image_processing_class.from_dict(self.image_processor_dict    ,					size=42						)
					self.assertEqual(image_processor.size    ,					{'shortest_edge': 42}						)





	def  _lowerCAmelCase    (			self							:					int						)	->  str:





					"""simple docstring"""


					pass





	def  _lowerCAmelCase    (			self							:					Tuple						)	->  Any:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       List[str] 							=    self.image_processing_class(**self.image_processor_dict						)
					# create random PIL images
					SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    prepare_image_inputs(self.image_processor_tester    ,					equal_resolution=_SCREAMING_SNAKE_CASE						)
					for image in image_inputs:
									self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,					Image.Image						)

					# Test not batched input
					SCREAMING_SNAKE_CASE	:       str 							=    image_processing(image_inputs[0]    ,					return_tensors='pt'						).pixel_values
					self.assertEqual(
					    encoded_images.shape    ,					(
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size['shortest_edge'],
					        self.image_processor_tester.size['shortest_edge'],
					    )    ,					)

					# Test batched
					SCREAMING_SNAKE_CASE	:       str 							=    image_processing(_SCREAMING_SNAKE_CASE    ,					return_tensors='pt'						).pixel_values
					self.assertEqual(
					    encoded_images.shape    ,					(
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size['shortest_edge'],
					        self.image_processor_tester.size['shortest_edge'],
					    )    ,					)





	def  _lowerCAmelCase    (			self							:					int						)	->  str:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Optional[int] 							=    self.image_processing_class(**self.image_processor_dict						)
					# create random numpy tensors
					SCREAMING_SNAKE_CASE	:       Tuple 							=    prepare_image_inputs(self.image_processor_tester    ,					equal_resolution=_SCREAMING_SNAKE_CASE    ,					numpify=_SCREAMING_SNAKE_CASE						)
					for image in image_inputs:
									self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,					np.ndarray						)

					# Test not batched input
					SCREAMING_SNAKE_CASE	:       int 							=    image_processing(image_inputs[0]    ,					return_tensors='pt'						).pixel_values
					self.assertEqual(
					    encoded_images.shape    ,					(
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size['shortest_edge'],
					        self.image_processor_tester.size['shortest_edge'],
					    )    ,					)

					# Test batched
					SCREAMING_SNAKE_CASE	:       Optional[int] 							=    image_processing(_SCREAMING_SNAKE_CASE    ,					return_tensors='pt'						).pixel_values
					self.assertEqual(
					    encoded_images.shape    ,					(
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size['shortest_edge'],
					        self.image_processor_tester.size['shortest_edge'],
					    )    ,					)







	def  _lowerCAmelCase    (			self							:					Optional[Any]						)	->  Dict:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       str 							=    self.image_processing_class(**self.image_processor_dict						)
					# create random PyTorch tensors
					SCREAMING_SNAKE_CASE	:       Tuple 							=    prepare_image_inputs(self.image_processor_tester    ,					equal_resolution=_SCREAMING_SNAKE_CASE    ,					torchify=_SCREAMING_SNAKE_CASE						)
					for image in image_inputs:
									self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,					torch.Tensor						)

					# Test not batched input
					SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    image_processing(image_inputs[0]    ,					return_tensors='pt'						).pixel_values
					self.assertEqual(
					    encoded_images.shape    ,					(
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size['shortest_edge'],
					        self.image_processor_tester.size['shortest_edge'],
					    )    ,					)

					# Test batched
					SCREAMING_SNAKE_CASE	:       Any 							=    image_processing(_SCREAMING_SNAKE_CASE    ,					return_tensors='pt'						).pixel_values
					self.assertEqual(
					    encoded_images.shape    ,					(
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size['shortest_edge'],
					        self.image_processor_tester.size['shortest_edge'],
					    )    ,					)





