


'''simple docstring'''







import itertools
import random
import unittest

import numpy as np

from transformers import EncodecFeatureExtractor
from transformers.testing_utils import require_torch
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
		import torch


_SCREAMING_SNAKE_CASE       :   Any        = random.Random()



def 						_UpperCamelCase							( UpperCamelCase__     ,   UpperCamelCase__=1.0     ,   UpperCamelCase__=None     ,   UpperCamelCase__=None		):






				"""simple docstring"""



				if rng is None:
								__magic_name__				:					Union[str, Any]						       =       global_rng

				__magic_name__				:					Tuple						       =       []
				for batch_idx in range(shape[0]		):
								values.append([]		)
								for _ in range(shape[1]		):
												values[-1].append(rng.random() * scale		)

				return values






@require_torch
class 		_snake_case    (				unittest.TestCase		):






						'''simple docstring'''







						def __init__(  self:	List[Any]      ,							__UpperCamelCase:	int      ,							__UpperCamelCase:	Optional[int]=7      ,							__UpperCamelCase:	Optional[int]=400      ,							__UpperCamelCase:	Dict=2000      ,							__UpperCamelCase:	Optional[Any]=1      ,							__UpperCamelCase:	Dict=0.0      ,							__UpperCamelCase:	int=2_4000      ,							__UpperCamelCase:	Optional[int]=True      ,							)							->	Any:
										__magic_name__				:					List[Any]						       =       parent
										__magic_name__				:					str						       =       batch_size
										__magic_name__				:					List[Any]						       =       min_seq_length
										__magic_name__				:					Tuple						       =       max_seq_length
										__magic_name__				:					Tuple						       =       (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
										__magic_name__				:					Optional[Any]						       =       feature_size
										__magic_name__				:					int						       =       padding_value
										__magic_name__				:					Any						       =       sampling_rate
										__magic_name__				:					str						       =       return_attention_mask







						def lowerCAmelCase__ (  self:	Union[str, Any]     )							->	Optional[int]:
										return {
										    "feature_size": self.feature_size,
										    "padding_value": self.padding_value,
										    "sampling_rate": self.sampling_rate,
										    "return_attention_mask": self.return_attention_mask,
										}







						def lowerCAmelCase__ (  self:	List[Any]      ,							__UpperCamelCase:	List[str]=False      ,							__UpperCamelCase:	int=False     )							->	Tuple:
										def _flatten(__UpperCamelCase:	Union[str, Any]     ):
														return list(itertools.chain(*__UpperCamelCase     )     )

										if equal_length:
														__magic_name__				:					Union[str, Any]						       =       floats_list((self.batch_size, self.max_seq_length)     )
										else:
														# make sure that inputs increase in size
														__magic_name__				:					Union[str, Any]						       =       [
														    _flatten(floats_list((x, self.feature_size)     )     )
														    for x in range(self.min_seq_length      ,							self.max_seq_length      ,							self.seq_length_diff     )
														]

										if numpify:
														__magic_name__				:					Tuple						       =       [np.asarray(__UpperCamelCase     ) for x in audio_inputs]

										return audio_inputs






@require_torch
class 		_snake_case    (				snake_case_  ,  unittest.TestCase		):






						'''simple docstring'''







						__snake_case   =      EncodecFeatureExtractor







						def lowerCAmelCase__ (  self:	List[str]     )							->	str:
										__magic_name__				:					List[Any]						       =       EnCodecFeatureExtractionTester(self     )







						def lowerCAmelCase__ (  self:	Any     )							->	Any:
										# Tests that all call wrap to encode_plus and batch_encode_plus
										__magic_name__				:					List[Any]						       =       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()     )
										# create three inputs of length 800, 1000, and 1200
										__magic_name__				:					Union[str, Any]						       =       [floats_list((1, x)     )[0] for x in range(800      ,							1400      ,							200     )]
										__magic_name__				:					Dict						       =       [np.asarray(__UpperCamelCase     ) for audio_input in audio_inputs]

										# Test not batched input
										__magic_name__				:					Tuple						       =       feat_extract(audio_inputs[0]      ,							return_tensors="np"     ).input_values
										__magic_name__				:					Optional[Any]						       =       feat_extract(np_audio_inputs[0]      ,							return_tensors="np"     ).input_values
										self.assertTrue(np.allclose(__UpperCamelCase      ,							__UpperCamelCase      ,							atol=1E-3     )     )

										# Test batched
										__magic_name__				:					Union[str, Any]						       =       feat_extract(__UpperCamelCase      ,							padding=__UpperCamelCase      ,							return_tensors="np"     ).input_values
										__magic_name__				:					List[Any]						       =       feat_extract(__UpperCamelCase      ,							padding=__UpperCamelCase      ,							return_tensors="np"     ).input_values
										for enc_seq_a, enc_seq_a in zip(__UpperCamelCase      ,							__UpperCamelCase     ):
														self.assertTrue(np.allclose(__UpperCamelCase      ,							__UpperCamelCase      ,							atol=1E-3     )     )







						def lowerCAmelCase__ (  self:	Optional[int]     )							->	int:
										__magic_name__				:					Optional[int]						       =       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()     )
										__magic_name__				:					Dict						       =       np.random.rand(100     ).astype(np.floataa     )
										__magic_name__				:					List[str]						       =       np_audio_inputs.tolist()

										for inputs in [py_audio_inputs, np_audio_inputs]:
														__magic_name__				:					Tuple						       =       feature_extractor.pad([{"input_values": inputs}]      ,							return_tensors="np"     )
														self.assertTrue(np_processed.input_values.dtype == np.floataa     )
														__magic_name__				:					List[str]						       =       feature_extractor.pad([{"input_values": inputs}]      ,							return_tensors="pt"     )
														self.assertTrue(pt_processed.input_values.dtype == torch.floataa     )







						def lowerCAmelCase__ (  self:	Optional[Any]      ,							__UpperCamelCase:	Dict     )							->	str:
										from datasets import load_dataset

										__magic_name__				:					Union[str, Any]						       =       load_dataset("hf-internal-testing/librispeech_asr_dummy"      ,							"clean"      ,							split="validation"     )
										# automatic decoding with librispeech
										__magic_name__				:					int						       =       ds.sort("id"     ).select(range(__UpperCamelCase     )     )[:num_samples]["audio"]

										return [x["array"] for x in audio_samples]







						def lowerCAmelCase__ (  self:	Union[str, Any]     )							->	List[Any]:
										# fmt: off
										__magic_name__				:					Union[str, Any]						       =       torch.tensor(
										    [2.3_804E-03, 2.0_752E-03, 1.9_836E-03, 2.1_057E-03, 1.6_174E-03,
										     3.0_518E-04, 9.1_553E-05, 3.3_569E-04, 9.7_656E-04, 1.8_311E-03,
										     2.0_142E-03, 2.1_057E-03, 1.7_395E-03, 4.5_776E-04, -3.9_673E-04,
										     4.5_776E-04, 1.0_071E-03, 9.1_553E-05, 4.8_828E-04, 1.1_597E-03,
										     7.3_242E-04, 9.4_604E-04, 1.8_005E-03, 1.8_311E-03, 8.8_501E-04,
										     4.2_725E-04, 4.8_828E-04, 7.3_242E-04, 1.0_986E-03, 2.1_057E-03]     )
										# fmt: on
										__magic_name__				:					int						       =       self._load_datasamples(1     )
										__magic_name__				:					List[Any]						       =       EncodecFeatureExtractor()
										__magic_name__				:					List[Any]						       =       feature_extractor(__UpperCamelCase      ,							return_tensors="pt"     ).input_values
										self.assertEquals(input_values.shape      ,							(1, 1, 9_3680)     )
										self.assertTrue(torch.allclose(input_values[0, 0, :30]      ,							__UpperCamelCase      ,							atol=1E-6     )     )







						def lowerCAmelCase__ (  self:	List[str]     )							->	List[str]:
										# fmt: off
										__magic_name__				:					Any						       =       torch.tensor(
										    [2.3_804E-03, 2.0_752E-03, 1.9_836E-03, 2.1_057E-03, 1.6_174E-03,
										     3.0_518E-04, 9.1_553E-05, 3.3_569E-04, 9.7_656E-04, 1.8_311E-03,
										     2.0_142E-03, 2.1_057E-03, 1.7_395E-03, 4.5_776E-04, -3.9_673E-04,
										     4.5_776E-04, 1.0_071E-03, 9.1_553E-05, 4.8_828E-04, 1.1_597E-03,
										     7.3_242E-04, 9.4_604E-04, 1.8_005E-03, 1.8_311E-03, 8.8_501E-04,
										     4.2_725E-04, 4.8_828E-04, 7.3_242E-04, 1.0_986E-03, 2.1_057E-03]     )
										# fmt: on
										__magic_name__				:					int						       =       self._load_datasamples(1     )
										__magic_name__				:					str						       =       [np.tile(input_audio[0][None]      ,							reps=(2, 1)     )]
										input_audio[0][1] *= 0.5
										__magic_name__				:					List[str]						       =       EncodecFeatureExtractor(feature_size=2     )
										__magic_name__				:					Union[str, Any]						       =       feature_extractor(__UpperCamelCase      ,							return_tensors="pt"     ).input_values
										self.assertEquals(input_values.shape      ,							(1, 2, 9_3680)     )
										self.assertTrue(torch.allclose(input_values[0, 0, :30]      ,							__UpperCamelCase      ,							atol=1E-6     )     )
										self.assertTrue(torch.allclose(input_values[0, 1, :30]      ,							EXPECTED_INPUT_VALUES * 0.5      ,							atol=1E-6     )     )







						def lowerCAmelCase__ (  self:	Tuple     )							->	Dict:
										__magic_name__				:					List[str]						       =       self._load_datasamples(2     )
										# would be easier if the stride was like
										__magic_name__				:					Optional[Any]						       =       EncodecFeatureExtractor(feature_size=1      ,							chunk_length_s=1      ,							overlap=0.0_1     )

										# pad and trunc raise an error ?
										with self.assertRaisesRegex(
										    __UpperCamelCase      ,							"^Both padding and truncation were set. Make sure you only set one.$"      ,							):
														__magic_name__				:					str						       =       feature_extractor(
														    __UpperCamelCase      ,							padding="max_length"      ,							truncation=__UpperCamelCase      ,							return_tensors="pt"     ).input_values

										# truncate to chunk
										__magic_name__				:					List[str]						       =       feature_extractor(__UpperCamelCase      ,							truncation=__UpperCamelCase      ,							return_tensors="pt"     ).input_values
										self.assertEquals(truncated_outputs.shape      ,							(2, 1, 7_1520)     )  # 2 chunks

										# force truncate to max_length
										__magic_name__				:					Tuple						       =       feature_extractor(
										    __UpperCamelCase      ,							truncation=__UpperCamelCase      ,							max_length=4_8000      ,							return_tensors="pt"     ).input_values
										self.assertEquals(truncated_outputs.shape      ,							(2, 1, 4_8000)     )

										# pad to chunk
										__magic_name__				:					Union[str, Any]						       =       feature_extractor(__UpperCamelCase      ,							padding=__UpperCamelCase      ,							return_tensors="pt"     ).input_values
										self.assertEquals(padded_outputs.shape      ,							(2, 1, 9_5280)     )

										# pad to chunk
										__magic_name__				:					int						       =       feature_extractor(__UpperCamelCase      ,							return_tensors="pt"     ).input_values
										self.assertEquals(truncated_outputs.shape      ,							(2, 1, 9_5280)     )

										# force pad to max length
										__magic_name__				:					Dict						       =       feature_extractor(
										    __UpperCamelCase      ,							padding="max_length"      ,							max_length=10_0000      ,							return_tensors="pt"     ).input_values
										self.assertEquals(truncated_outputs.shape      ,							(2, 1, 10_0000)     )

										# force no pad
										with self.assertRaisesRegex(
										    __UpperCamelCase      ,							"^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"      ,							):
														__magic_name__				:					Dict						       =       feature_extractor(__UpperCamelCase      ,							padding=__UpperCamelCase      ,							return_tensors="pt"     ).input_values

										__magic_name__				:					str						       =       feature_extractor(input_audio[0]      ,							padding=__UpperCamelCase      ,							return_tensors="pt"     ).input_values
										self.assertEquals(truncated_outputs.shape      ,							(1, 1, 9_3680)     )

										# no pad if no chunk_length_s
										__magic_name__				:					int						       =       None
										with self.assertRaisesRegex(
										    __UpperCamelCase      ,							"^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"      ,							):
														__magic_name__				:					Dict						       =       feature_extractor(__UpperCamelCase      ,							padding=__UpperCamelCase      ,							return_tensors="pt"     ).input_values

										__magic_name__				:					List[str]						       =       feature_extractor(input_audio[0]      ,							padding=__UpperCamelCase      ,							return_tensors="pt"     ).input_values
										self.assertEquals(truncated_outputs.shape      ,							(1, 1, 9_3680)     )

										# no pad if no overlap
										__magic_name__				:					Any						       =       2
										__magic_name__				:					Any						       =       None
										with self.assertRaisesRegex(
										    __UpperCamelCase      ,							"^Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.$"      ,							):
														__magic_name__				:					Any						       =       feature_extractor(__UpperCamelCase      ,							padding=__UpperCamelCase      ,							return_tensors="pt"     ).input_values

										__magic_name__				:					Optional[Any]						       =       feature_extractor(input_audio[0]      ,							padding=__UpperCamelCase      ,							return_tensors="pt"     ).input_values
										self.assertEquals(truncated_outputs.shape      ,							(1, 1, 9_3680)     )



'''simple docstring'''







import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 		_snake_case    (				snake_case_  ,  unittest.TestCase		):






						'''simple docstring'''







						__snake_case   =      GPTNeoXJapaneseTokenizer
						__snake_case   =      False
						__snake_case   =      {"do_clean_text": False, "add_prefix_space": False}







						def lowerCAmelCase__ (  self:	Any     )							->	List[str]:
										super().setUp()

										__magic_name__				:					Union[str, Any]						       =       [
										    "こん",
										    "こんに",
										    "にちは",
										    "ばんは",
										    "世界,㔺界",
										    "、",
										    "。",
										    "<BR>",
										    "<SP>",
										    "<TAB>",
										    "<URL>",
										    "<EMAIL>",
										    "<TEL>",
										    "<DATE>",
										    "<PRICE>",
										    "<BLOCK>",
										    "<KIGOU>",
										    "<U2000U2BFF>",
										    "<|emoji1|>",
										    "<unk>",
										    "<|startoftext|>",
										    "<|endoftext|>",
										]
										__magic_name__				:					List[str]						       =       {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
										__magic_name__				:					str						       =       {"unk_token": "<unk>"}

										__magic_name__				:					Dict						       =       os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["vocab_file"]     )
										__magic_name__				:					Dict						       =       os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["emoji_file"]     )
										with open(self.vocab_file      ,							"w"      ,							encoding="utf-8"     ) as vocab_writer:
														vocab_writer.write("".join([x + "\n" for x in vocab_tokens]     )     )
										with open(self.emoji_file      ,							"w"     ) as emoji_writer:
														emoji_writer.write(json.dumps(__UpperCamelCase     )     )







						def lowerCAmelCase__ (  self:	Tuple      ,							**__UpperCamelCase:	str     )							->	Union[str, Any]:
										kwargs.update(self.special_tokens_map     )
										return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname      ,							**__UpperCamelCase     )







						def lowerCAmelCase__ (  self:	List[str]      ,							__UpperCamelCase:	str     )							->	List[str]:
										__magic_name__				:					str						       =       "こんにちは、世界。 \nこんばんは、㔺界。😀"
										__magic_name__				:					str						       =       "こんにちは、世界。 \nこんばんは、世界。😀"
										return input_text, output_text







						def lowerCAmelCase__ (  self:	Union[str, Any]      ,							__UpperCamelCase:	List[Any]     )							->	Dict:
										__magic_name__       ,     __magic_name__				:					Union[str, Any]						       =       self.get_input_output_texts(__UpperCamelCase     )
										__magic_name__				:					List[Any]						       =       tokenizer.encode(__UpperCamelCase      ,							add_special_tokens=__UpperCamelCase     )
										__magic_name__				:					Optional[Any]						       =       tokenizer.decode(__UpperCamelCase      ,							clean_up_tokenization_spaces=__UpperCamelCase     )
										return text, ids







						def lowerCAmelCase__ (  self:	Optional[int]     )							->	Union[str, Any]:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	str     )							->	List[str]:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	str     )							->	str:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	Union[str, Any]     )							->	Dict:
										__magic_name__				:					List[str]						       =       self.get_tokenizer()

										# Testing tokenization
										__magic_name__				:					Any						       =       "こんにちは、世界。　こんばんは、㔺界。"
										__magic_name__				:					Optional[int]						       =       ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
										__magic_name__				:					List[Any]						       =       tokenizer.tokenize(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )

										# Testing conversion to ids without special tokens
										__magic_name__				:					int						       =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
										__magic_name__				:					str						       =       tokenizer.convert_tokens_to_ids(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )

										# Testing conversion to ids with special tokens
										__magic_name__				:					str						       =       tokens + [tokenizer.unk_token]
										__magic_name__				:					Tuple						       =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
										__magic_name__				:					Dict						       =       tokenizer.convert_tokens_to_ids(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )







						@slow
						def lowerCAmelCase__ (  self:	List[Any]     )							->	Union[str, Any]:
										__magic_name__				:					List[str]						       =       self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b"     )

										__magic_name__				:					List[Any]						       =       tokenizer.encode("ありがとう。"      ,							add_special_tokens=__UpperCamelCase     )
										__magic_name__				:					Optional[Any]						       =       tokenizer.encode("どういたしまして。"      ,							add_special_tokens=__UpperCamelCase     )

										__magic_name__				:					List[Any]						       =       tokenizer.build_inputs_with_special_tokens(__UpperCamelCase     )
										__magic_name__				:					str						       =       tokenizer.build_inputs_with_special_tokens(__UpperCamelCase      ,							__UpperCamelCase     )

										assert encoded_sentence == ids_a
										assert encoded_pair == ids_a + ids_a







						def lowerCAmelCase__ (  self:	Tuple     )							->	Dict:
										# Intentionally convert some words to accommodate character fluctuations unique to Japanese
										pass







						def lowerCAmelCase__ (  self:	Dict     )							->	Optional[int]:
										# tokenizer has no padding token
										pass