


'''simple docstring'''







from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
		import PIL


_SCREAMING_SNAKE_CASE       :   Tuple        = logging.get_logger(__name__)






class 		_snake_case    (				__lowerCamelCase		):






						'''simple docstring'''







						__snake_case   =      ['pixel_values']







						def __init__(  self:	List[str]      ,							__UpperCamelCase:	bool = True      ,							__UpperCamelCase:	Dict[str, int] = None      ,							__UpperCamelCase:	PILImageResampling = PIL.Image.NEAREST      ,							__UpperCamelCase:	bool = False      ,							__UpperCamelCase:	Dict[str, int] = None      ,							__UpperCamelCase:	Union[int, float] = 1 / 255      ,							__UpperCamelCase:	bool = False      ,							__UpperCamelCase:	bool = True      ,							__UpperCamelCase:	bool = True      ,							__UpperCamelCase:	Optional[Union[float, List[float]]] = None      ,							__UpperCamelCase:	Optional[Union[float, List[float]]] = None      ,							__UpperCamelCase:	bool = True      ,							**__UpperCamelCase:	Optional[Any]      ,							)							->	None:
										super().__init__(**__UpperCamelCase     )
										__magic_name__				:					Optional[Any]						       =       size if size is not None else {"height": 346, "width": 346}
										__magic_name__				:					List[str]						       =       get_size_dict(__UpperCamelCase     )
										__magic_name__				:					List[Any]						       =       crop_size if crop_size is not None else {"height": 289, "width": 289}
										__magic_name__				:					Union[str, Any]						       =       get_size_dict(__UpperCamelCase      ,							param_name="crop_size"     )

										__magic_name__				:					Union[str, Any]						       =       do_resize
										__magic_name__				:					Tuple						       =       size
										__magic_name__				:					List[Any]						       =       resample
										__magic_name__				:					Optional[Any]						       =       do_center_crop
										__magic_name__				:					Dict						       =       crop_size
										__magic_name__				:					Optional[Any]						       =       do_rescale
										__magic_name__				:					Tuple						       =       rescale_factor
										__magic_name__				:					Union[str, Any]						       =       rescale_offset
										__magic_name__				:					Dict						       =       do_normalize
										__magic_name__				:					Union[str, Any]						       =       image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
										__magic_name__				:					Tuple						       =       image_std if image_std is not None else IMAGENET_STANDARD_STD
										__magic_name__				:					Optional[int]						       =       include_top







						def lowerCAmelCase__ (  self:	List[Any]      ,							__UpperCamelCase:	np.ndarray      ,							__UpperCamelCase:	Dict[str, int]      ,							__UpperCamelCase:	PILImageResampling = PIL.Image.NEAREST      ,							__UpperCamelCase:	Optional[Union[str, ChannelDimension]] = None      ,							**__UpperCamelCase:	int      ,							)							->	np.ndarray:
										__magic_name__				:					int						       =       get_size_dict(__UpperCamelCase     )
										if "height" not in size or "width" not in size:
														raise ValueError(f"""The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"""     )
										return resize(
										    __UpperCamelCase      ,							size=(size["height"], size["width"])      ,							resample=__UpperCamelCase      ,							data_format=__UpperCamelCase      ,							**__UpperCamelCase     )







						def lowerCAmelCase__ (  self:	int      ,							__UpperCamelCase:	np.ndarray      ,							__UpperCamelCase:	Dict[str, int]      ,							__UpperCamelCase:	Optional[Union[str, ChannelDimension]] = None      ,							**__UpperCamelCase:	Optional[Any]      ,							)							->	np.ndarray:
										__magic_name__				:					List[str]						       =       get_size_dict(__UpperCamelCase     )
										if "height" not in size or "width" not in size:
														raise ValueError(f"""The size dictionary must have keys 'height' and 'width'. Got {size.keys()}"""     )
										return center_crop(__UpperCamelCase      ,							size=(size["height"], size["width"])      ,							data_format=__UpperCamelCase      ,							**__UpperCamelCase     )







						def lowerCAmelCase__ (  self:	Any      ,							__UpperCamelCase:	np.ndarray      ,							__UpperCamelCase:	Union[int, float]      ,							__UpperCamelCase:	bool = True      ,							__UpperCamelCase:	Optional[Union[str, ChannelDimension]] = None      ,							**__UpperCamelCase:	Union[str, Any]      ,							)							->	Optional[int]:






										if offset:
														__magic_name__				:					Dict						       =       (image - 127.5) * scale
														if data_format is not None:
																		__magic_name__				:					List[Any]						       =       to_channel_dimension_format(__UpperCamelCase      ,							__UpperCamelCase     )
														__magic_name__				:					Optional[Any]						       =       rescaled_image.astype(np.floataa     )
										else:
														__magic_name__				:					Dict						       =       rescale(__UpperCamelCase      ,							scale=__UpperCamelCase      ,							data_format=__UpperCamelCase      ,							**__UpperCamelCase     )
										return rescaled_image







						def lowerCAmelCase__ (  self:	Union[str, Any]      ,							__UpperCamelCase:	np.ndarray      ,							__UpperCamelCase:	Union[float, List[float]]      ,							__UpperCamelCase:	Union[float, List[float]]      ,							__UpperCamelCase:	Optional[Union[str, ChannelDimension]] = None      ,							**__UpperCamelCase:	int      ,							)							->	np.ndarray:
										return normalize(__UpperCamelCase      ,							mean=__UpperCamelCase      ,							std=__UpperCamelCase      ,							data_format=__UpperCamelCase      ,							**__UpperCamelCase     )







						def lowerCAmelCase__ (  self:	Dict      ,							__UpperCamelCase:	ImageInput      ,							__UpperCamelCase:	bool = None      ,							__UpperCamelCase:	Dict[str, int] = None      ,							__UpperCamelCase:	Optional[int]=None      ,							__UpperCamelCase:	bool = None      ,							__UpperCamelCase:	Dict[str, int] = None      ,							__UpperCamelCase:	bool = None      ,							__UpperCamelCase:	float = None      ,							__UpperCamelCase:	bool = None      ,							__UpperCamelCase:	bool = None      ,							__UpperCamelCase:	Optional[Union[float, List[float]]] = None      ,							__UpperCamelCase:	Optional[Union[float, List[float]]] = None      ,							__UpperCamelCase:	bool = None      ,							__UpperCamelCase:	Optional[Union[str, TensorType]] = None      ,							__UpperCamelCase:	ChannelDimension = ChannelDimension.FIRST      ,							**__UpperCamelCase:	Dict      ,							)							->	PIL.Image.Image:
										__magic_name__				:					List[str]						       =       do_resize if do_resize is not None else self.do_resize
										__magic_name__				:					List[str]						       =       resample if resample is not None else self.resample
										__magic_name__				:					Union[str, Any]						       =       do_center_crop if do_center_crop is not None else self.do_center_crop
										__magic_name__				:					Union[str, Any]						       =       do_rescale if do_rescale is not None else self.do_rescale
										__magic_name__				:					Union[str, Any]						       =       rescale_factor if rescale_factor is not None else self.rescale_factor
										__magic_name__				:					List[Any]						       =       rescale_offset if rescale_offset is not None else self.rescale_offset
										__magic_name__				:					int						       =       do_normalize if do_normalize is not None else self.do_normalize
										__magic_name__				:					List[str]						       =       image_mean if image_mean is not None else self.image_mean
										__magic_name__				:					Dict						       =       image_std if image_std is not None else self.image_std
										__magic_name__				:					List[str]						       =       include_top if include_top is not None else self.include_top

										__magic_name__				:					Optional[Any]						       =       size if size is not None else self.size
										__magic_name__				:					Union[str, Any]						       =       get_size_dict(__UpperCamelCase     )
										__magic_name__				:					List[str]						       =       crop_size if crop_size is not None else self.crop_size
										__magic_name__				:					Optional[int]						       =       get_size_dict(__UpperCamelCase      ,							param_name="crop_size"     )

										__magic_name__				:					List[Any]						       =       make_list_of_images(__UpperCamelCase     )

										if not valid_images(__UpperCamelCase     ):
														raise ValueError(
														    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
														    "torch.Tensor, tf.Tensor or jax.ndarray."     )

										if do_resize and size is None or resample is None:
														raise ValueError("Size and resample must be specified if do_resize is True."     )

										if do_center_crop and crop_size is None:
														raise ValueError("Crop size must be specified if do_center_crop is True."     )

										if do_rescale and rescale_factor is None:
														raise ValueError("Rescale factor must be specified if do_rescale is True."     )

										if do_normalize and (image_mean is None or image_std is None):
														raise ValueError("Image mean and std must be specified if do_normalize is True."     )

										# All transformations expect numpy arrays.
										__magic_name__				:					str						       =       [to_numpy_array(__UpperCamelCase     ) for image in images]

										if do_resize:
														__magic_name__				:					Any						       =       [self.resize(image=__UpperCamelCase      ,							size=__UpperCamelCase      ,							resample=__UpperCamelCase     ) for image in images]

										if do_center_crop:
														__magic_name__				:					Any						       =       [self.center_crop(image=__UpperCamelCase      ,							size=__UpperCamelCase     ) for image in images]

										if do_rescale:
														__magic_name__				:					str						       =       [self.rescale(image=__UpperCamelCase      ,							scale=__UpperCamelCase      ,							offset=__UpperCamelCase     ) for image in images]

										if do_normalize:
														__magic_name__				:					Any						       =       [self.normalize(image=__UpperCamelCase      ,							mean=__UpperCamelCase      ,							std=__UpperCamelCase     ) for image in images]

										if include_top:
														__magic_name__				:					str						       =       [self.normalize(image=__UpperCamelCase      ,							mean=[0, 0, 0]      ,							std=__UpperCamelCase     ) for image in images]

										__magic_name__				:					Optional[int]						       =       [to_channel_dimension_format(__UpperCamelCase      ,							__UpperCamelCase     ) for image in images]

										__magic_name__				:					str						       =       {"pixel_values": images}
										return BatchFeature(data=__UpperCamelCase      ,							tensor_type=__UpperCamelCase     )



'''simple docstring'''







import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 		_snake_case    (				snake_case_  ,  unittest.TestCase		):






						'''simple docstring'''







						__snake_case   =      GPTNeoXJapaneseTokenizer
						__snake_case   =      False
						__snake_case   =      {"do_clean_text": False, "add_prefix_space": False}







						def lowerCAmelCase__ (  self:	Any     )							->	List[str]:
										super().setUp()

										__magic_name__				:					Union[str, Any]						       =       [
										    "こん",
										    "こんに",
										    "にちは",
										    "ばんは",
										    "世界,㔺界",
										    "、",
										    "。",
										    "<BR>",
										    "<SP>",
										    "<TAB>",
										    "<URL>",
										    "<EMAIL>",
										    "<TEL>",
										    "<DATE>",
										    "<PRICE>",
										    "<BLOCK>",
										    "<KIGOU>",
										    "<U2000U2BFF>",
										    "<|emoji1|>",
										    "<unk>",
										    "<|startoftext|>",
										    "<|endoftext|>",
										]
										__magic_name__				:					List[str]						       =       {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
										__magic_name__				:					str						       =       {"unk_token": "<unk>"}

										__magic_name__				:					Dict						       =       os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["vocab_file"]     )
										__magic_name__				:					Dict						       =       os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["emoji_file"]     )
										with open(self.vocab_file      ,							"w"      ,							encoding="utf-8"     ) as vocab_writer:
														vocab_writer.write("".join([x + "\n" for x in vocab_tokens]     )     )
										with open(self.emoji_file      ,							"w"     ) as emoji_writer:
														emoji_writer.write(json.dumps(__UpperCamelCase     )     )







						def lowerCAmelCase__ (  self:	Tuple      ,							**__UpperCamelCase:	str     )							->	Union[str, Any]:
										kwargs.update(self.special_tokens_map     )
										return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname      ,							**__UpperCamelCase     )







						def lowerCAmelCase__ (  self:	List[str]      ,							__UpperCamelCase:	str     )							->	List[str]:
										__magic_name__				:					str						       =       "こんにちは、世界。 \nこんばんは、㔺界。😀"
										__magic_name__				:					str						       =       "こんにちは、世界。 \nこんばんは、世界。😀"
										return input_text, output_text







						def lowerCAmelCase__ (  self:	Union[str, Any]      ,							__UpperCamelCase:	List[Any]     )							->	Dict:
										__magic_name__       ,     __magic_name__				:					Union[str, Any]						       =       self.get_input_output_texts(__UpperCamelCase     )
										__magic_name__				:					List[Any]						       =       tokenizer.encode(__UpperCamelCase      ,							add_special_tokens=__UpperCamelCase     )
										__magic_name__				:					Optional[Any]						       =       tokenizer.decode(__UpperCamelCase      ,							clean_up_tokenization_spaces=__UpperCamelCase     )
										return text, ids







						def lowerCAmelCase__ (  self:	Optional[int]     )							->	Union[str, Any]:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	str     )							->	List[str]:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	str     )							->	str:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	Union[str, Any]     )							->	Dict:
										__magic_name__				:					List[str]						       =       self.get_tokenizer()

										# Testing tokenization
										__magic_name__				:					Any						       =       "こんにちは、世界。　こんばんは、㔺界。"
										__magic_name__				:					Optional[int]						       =       ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
										__magic_name__				:					List[Any]						       =       tokenizer.tokenize(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )

										# Testing conversion to ids without special tokens
										__magic_name__				:					int						       =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
										__magic_name__				:					str						       =       tokenizer.convert_tokens_to_ids(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )

										# Testing conversion to ids with special tokens
										__magic_name__				:					str						       =       tokens + [tokenizer.unk_token]
										__magic_name__				:					Tuple						       =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
										__magic_name__				:					Dict						       =       tokenizer.convert_tokens_to_ids(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )







						@slow
						def lowerCAmelCase__ (  self:	List[Any]     )							->	Union[str, Any]:
										__magic_name__				:					List[str]						       =       self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b"     )

										__magic_name__				:					List[Any]						       =       tokenizer.encode("ありがとう。"      ,							add_special_tokens=__UpperCamelCase     )
										__magic_name__				:					Optional[Any]						       =       tokenizer.encode("どういたしまして。"      ,							add_special_tokens=__UpperCamelCase     )

										__magic_name__				:					List[Any]						       =       tokenizer.build_inputs_with_special_tokens(__UpperCamelCase     )
										__magic_name__				:					str						       =       tokenizer.build_inputs_with_special_tokens(__UpperCamelCase      ,							__UpperCamelCase     )

										assert encoded_sentence == ids_a
										assert encoded_pair == ids_a + ids_a







						def lowerCAmelCase__ (  self:	Tuple     )							->	Dict:
										# Intentionally convert some words to accommodate character fluctuations unique to Japanese
										pass







						def lowerCAmelCase__ (  self:	Dict     )							->	Optional[int]:
										# tokenizer has no padding token
										pass