




'''simple docstring'''






from __future__ import annotations

import unittest

from transformers import CTRLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers.models.ctrl.modeling_tf_ctrl import (
						    TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFCTRLForSequenceClassification,
						    TFCTRLLMHeadModel,
						    TFCTRLModel,
						)






class 					lowercase_			(	lowerCAmelCase_       ):





					def __init__(    self    :				Union[str, Any]    ,       __lowerCamelCase    :				Optional[int]    ,       ):
											snake_case__					:      List[Any]  			=						parent
											snake_case__					:      List[Any]  			=						13
											snake_case__					:      Optional[int]  			=						7
											snake_case__					:      Any  			=						True
											snake_case__					:      str  			=						True
											snake_case__					:      int  			=						True
											snake_case__					:      Any  			=						True
											snake_case__					:      Optional[Any]  			=						True
											snake_case__					:      Optional[int]  			=						99
											snake_case__					:      Optional[int]  			=						32
											snake_case__					:      List[str]  			=						2
											snake_case__					:      str  			=						4
											snake_case__					:      Optional[int]  			=						37
											snake_case__					:      Union[str, Any]  			=						'gelu'
											snake_case__					:      List[str]  			=						0.1
											snake_case__					:      List[str]  			=						0.1
											snake_case__					:      Tuple  			=						512
											snake_case__					:      Any  			=						16
											snake_case__					:      Optional[Any]  			=						2
											snake_case__					:      Any  			=						0.0_2
											snake_case__					:      int  			=						3
											snake_case__					:      Union[str, Any]  			=						4
											snake_case__					:      Optional[int]  			=						None
											snake_case__					:      Any  			=						self.vocab_size - 1





					def  _lowerCAmelCase			(    self    :				List[Any] ):
											snake_case__					:      Dict  			=						ids_tensor([self.batch_size, self.seq_length]    ,       self.vocab_size )

											snake_case__					:      Any  			=						None
											if self.use_input_mask:
																	snake_case__					:      Optional[int]  			=						random_attention_mask([self.batch_size, self.seq_length] )

											snake_case__					:      Any  			=						None
											if self.use_token_type_ids:
																	snake_case__					:      List[str]  			=						ids_tensor([self.batch_size, self.seq_length]    ,       self.type_vocab_size )

											snake_case__					:      Tuple  			=						None
											if self.use_mc_token_ids:
																	snake_case__					:      Union[str, Any]  			=						ids_tensor([self.batch_size, self.num_choices]    ,       self.seq_length )

											snake_case__					:      List[str]  			=						None
											snake_case__					:      List[Any]  			=						None
											snake_case__					:      List[str]  			=						None
											if self.use_labels:
																	snake_case__					:      Dict  			=						ids_tensor([self.batch_size]    ,       self.type_sequence_label_size )
																	snake_case__					:      List[Any]  			=						ids_tensor([self.batch_size, self.seq_length]    ,       self.num_labels )
																	snake_case__					:      Optional[Any]  			=						ids_tensor([self.batch_size]    ,       self.num_choices )

											snake_case__					:      Dict  			=						CTRLConfig(
											    vocab_size=self.vocab_size    ,       n_embd=self.hidden_size    ,       n_layer=self.num_hidden_layers    ,       n_head=self.num_attention_heads    ,       n_positions=self.max_position_embeddings    ,       pad_token_id=self.pad_token_id    ,       )

											snake_case__					:      Union[str, Any]  			=						ids_tensor([self.num_hidden_layers, self.num_attention_heads]    ,       2 )

											return (
											    config,
											    input_ids,
											    input_mask,
											    head_mask,
											    token_type_ids,
											    mc_token_ids,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											)





					def  _lowerCAmelCase			(    self    :				List[Any]    ,       __lowerCamelCase    :				List[Any]    ,       __lowerCamelCase    :				List[Any]    ,       __lowerCamelCase    :				Union[str, Any]    ,       __lowerCamelCase    :				Optional[Any]    ,       __lowerCamelCase    :				Tuple    ,       *__lowerCamelCase    :				Optional[int] ):
											snake_case__					:      Optional[Any]  			=						TFCTRLModel(config=__lowerCamelCase )
											snake_case__					:      List[str]  			=						{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
											snake_case__					:      Tuple  			=						model(__lowerCamelCase )

											snake_case__					:      List[str]  			=						[input_ids, None, input_mask]  # None is the input for 'past'
											snake_case__					:      Union[str, Any]  			=						model(__lowerCamelCase )

											snake_case__					:      Dict  			=						model(__lowerCamelCase )

											self.parent.assertEqual(result.last_hidden_state.shape    ,       (self.batch_size, self.seq_length, self.hidden_size) )





					def  _lowerCAmelCase			(    self    :				Optional[Any]    ,       __lowerCamelCase    :				List[str]    ,       __lowerCamelCase    :				Any    ,       __lowerCamelCase    :				Optional[Any]    ,       __lowerCamelCase    :				Any    ,       __lowerCamelCase    :				int    ,       *__lowerCamelCase    :				Any ):
											snake_case__					:      int  			=						TFCTRLLMHeadModel(config=__lowerCamelCase )
											snake_case__					:      List[str]  			=						{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
											snake_case__					:      Union[str, Any]  			=						model(__lowerCamelCase )
											self.parent.assertEqual(result.logits.shape    ,       (self.batch_size, self.seq_length, self.vocab_size) )





					def  _lowerCAmelCase			(    self    :				Dict    ,       __lowerCamelCase    :				Any    ,       __lowerCamelCase    :				List[Any]    ,       __lowerCamelCase    :				str    ,       __lowerCamelCase    :				Optional[int]    ,       __lowerCamelCase    :				str    ,       *__lowerCamelCase    :				Optional[int] ):
											snake_case__					:      List[Any]  			=						self.num_labels
											snake_case__					:      str  			=						ids_tensor([self.batch_size]    ,       self.type_sequence_label_size )
											snake_case__					:      Any  			=						{
											    'input_ids': input_ids,
											    'token_type_ids': token_type_ids,
											    'labels': sequence_labels,
											}
											snake_case__					:      Dict  			=						TFCTRLForSequenceClassification(__lowerCamelCase )
											snake_case__					:      Union[str, Any]  			=						model(__lowerCamelCase )
											self.parent.assertEqual(result.logits.shape    ,       (self.batch_size, self.num_labels) )





					def  _lowerCAmelCase			(    self    :				List[Any] ):
											snake_case__					:      List[str]  			=						self.prepare_config_and_inputs()

											(
											    snake_case__       
											)					:      int  			=						config_and_inputs

											snake_case__					:      Tuple  			=						{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
											return config, inputs_dict






@require_tf
class 					lowercase_			(	lowerCAmelCase_	,					lowerCAmelCase_	,					unittest.TestCase       ):
					A_       							= (TFCTRLModel, TFCTRLLMHeadModel, TFCTRLForSequenceClassification) if is_tf_available() else ()
					A_       							= (TFCTRLLMHeadModel,) if is_tf_available() else ()
					A_       							= (
					    {
					        "feature-extraction": TFCTRLModel,
					        "text-classification": TFCTRLForSequenceClassification,
					        "text-generation": TFCTRLLMHeadModel,
					        "zero-shot": TFCTRLForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					A_       							= False
					A_       							= False





					def  _lowerCAmelCase			(    self    :				int    ,       __lowerCamelCase    :				Tuple    ,       __lowerCamelCase    :				Tuple    ,       __lowerCamelCase    :				Dict    ,       __lowerCamelCase    :				str    ,       __lowerCamelCase    :				List[str] ):
											if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
																	# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
																	# `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
																	# config could not be created.
																	return True

											return False





					def  _lowerCAmelCase			(    self    :				Optional[int] ):
											snake_case__					:      List[Any]  			=						TFCTRLModelTester(self )
											snake_case__					:      Tuple  			=						ConfigTester(self    ,       config_class=__lowerCamelCase    ,       n_embd=37 )





					def  _lowerCAmelCase			(    self    :				Union[str, Any] ):
											self.config_tester.run_common_tests()





					def  _lowerCAmelCase			(    self    :				Any ):
											snake_case__					:      Optional[Any]  			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_ctrl_model(*__lowerCamelCase )





					def  _lowerCAmelCase			(    self    :				Any ):
											snake_case__					:      str  			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_ctrl_lm_head(*__lowerCamelCase )





					def  _lowerCAmelCase			(    self    :				Optional[int] ):
											snake_case__					:      Any  			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_ctrl_for_sequence_classification(*__lowerCamelCase )





					def  _lowerCAmelCase			(    self    :				List[str] ):
											snake_case__					:      str  			=						self.model_tester.prepare_config_and_inputs_for_common()
											snake_case__					:      Optional[Any]  			=						[TFCTRLLMHeadModel]
											snake_case__					:      Optional[int]  			=						[TFCTRLForSequenceClassification]

											for model_class in self.all_model_classes:
																	snake_case__					:      Dict  			=						model_class(__lowerCamelCase )
																	model.build()  # may be needed for the get_bias() call below
																	assert isinstance(model.get_input_embeddings()    ,       tf.keras.layers.Layer )

																	if model_class in list_lm_models:
																							snake_case__					:      List[str]  			=						model.get_output_embeddings()
																							assert isinstance(__lowerCamelCase    ,       tf.keras.layers.Layer )
																							snake_case__					:      Any  			=						model.get_bias()
																							assert isinstance(__lowerCamelCase    ,       __lowerCamelCase )
																							for k, v in name.items():
																													assert isinstance(__lowerCamelCase    ,       tf.Variable )
																	elif model_class in list_other_models_with_output_ebd:
																							snake_case__					:      Any  			=						model.get_output_embeddings()
																							assert isinstance(__lowerCamelCase    ,       tf.keras.layers.Layer )
																							snake_case__					:      int  			=						model.get_bias()
																							assert name is None
																	else:
																							snake_case__					:      Tuple  			=						model.get_output_embeddings()
																							assert x is None
																							snake_case__					:      Optional[int]  			=						model.get_bias()
																							assert name is None





					@slow
					def  _lowerCAmelCase			(    self    :				Any ):
											for model_name in TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	snake_case__					:      List[str]  			=						TFCTRLModel.from_pretrained(__lowerCamelCase )
																	self.assertIsNotNone(__lowerCamelCase )






@require_tf
class 					lowercase_			(	unittest.TestCase       ):





					@slow
					def  _lowerCAmelCase			(    self    :				Tuple ):
											snake_case__					:      str  			=						TFCTRLLMHeadModel.from_pretrained('ctrl' )
											snake_case__					:      List[Any]  			=						tf.convert_to_tensor([[11859, 0, 1611, 8]]    ,       dtype=tf.intaa )  # Legal the president is
											snake_case__					:      Optional[Any]  			=						[
											    11859,
											    0,
											    1611,
											    8,
											    5,
											    150,
											    26449,
											    2,
											    19,
											    348,
											    469,
											    3,
											    2595,
											    48,
											    20740,
											    246533,
											    246533,
											    19,
											    30,
											    5,
											]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

											snake_case__					:      List[Any]  			=						model.generate(__lowerCamelCase    ,       do_sample=__lowerCamelCase )
											self.assertListEqual(output_ids[0].numpy().tolist()    ,       __lowerCamelCase )






'''simple docstring'''






import shutil
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SpeechaTextTokenizer, is_speech_available
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_speech_to_text import floats_list


if is_speech_available():
      from transformers import SpeechaTextFeatureExtractor, SpeechaTextProcessor


A_ 			=		get_tests_dir("fixtures/test_sentencepiece.model")






@require_torch
@require_torchaudio
@require_sentencepiece
class 					lowercase_			(	unittest.TestCase       ):





     def  _lowerCAmelCase			(    self    :				List[str] ):
           snake_case__					:      Dict  			=						tempfile.mkdtemp()

           snake_case__					:      Union[str, Any]  			=						['<s>', '<pad>', '</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est']
           snake_case__					:      str  			=						dict(zip(__lowerCamelCase    ,       range(len(__lowerCamelCase ) ) ) )
           snake_case__					:      Union[str, Any]  			=						Path(self.tmpdirname )
           save_json(__lowerCamelCase    ,       save_dir / VOCAB_FILES_NAMES['vocab_file'] )
           if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
                 copyfile(__lowerCamelCase    ,       save_dir / VOCAB_FILES_NAMES['spm_file'] )

           snake_case__					:      Optional[Any]  			=						SpeechaTextTokenizer.from_pretrained(self.tmpdirname )
           tokenizer.save_pretrained(self.tmpdirname )

           snake_case__					:      str  			=						{
               'feature_size': 24,
               'num_mel_bins': 24,
               'padding_value': 0.0,
               'sampling_rate': 16000,
               'return_attention_mask': False,
               'do_normalize': True,
           }
           save_json(__lowerCamelCase    ,       save_dir / FEATURE_EXTRACTOR_NAME )





     def  _lowerCAmelCase			(    self    :				str    ,       **__lowerCamelCase    :				int ):
           return SpeechaTextTokenizer.from_pretrained(self.tmpdirname    ,       **__lowerCamelCase )





     def  _lowerCAmelCase			(    self    :				int    ,       **__lowerCamelCase    :				List[Any] ):
           return SpeechaTextFeatureExtractor.from_pretrained(self.tmpdirname    ,       **__lowerCamelCase )





     def  _lowerCAmelCase			(    self    :				List[Any] ):
           shutil.rmtree(self.tmpdirname )





     def  _lowerCAmelCase			(    self    :				Dict ):
           snake_case__					:      Dict  			=						self.get_tokenizer()
           snake_case__					:      Any  			=						self.get_feature_extractor()

           snake_case__					:      Tuple  			=						SpeechaTextProcessor(tokenizer=__lowerCamelCase    ,       feature_extractor=__lowerCamelCase )

           processor.save_pretrained(self.tmpdirname )
           snake_case__					:      Any  			=						SpeechaTextProcessor.from_pretrained(self.tmpdirname )

           self.assertEqual(processor.tokenizer.get_vocab()    ,       tokenizer.get_vocab() )
           self.assertIsInstance(processor.tokenizer    ,       __lowerCamelCase )

           self.assertEqual(processor.feature_extractor.to_json_string()    ,       feature_extractor.to_json_string() )
           self.assertIsInstance(processor.feature_extractor    ,       __lowerCamelCase )





     def  _lowerCAmelCase			(    self    :				List[Any] ):
           snake_case__					:      Any  			=						SpeechaTextProcessor(
               tokenizer=self.get_tokenizer()    ,       feature_extractor=self.get_feature_extractor() )
           processor.save_pretrained(self.tmpdirname )

           snake_case__					:      Tuple  			=						self.get_tokenizer(bos_token='(BOS)'    ,       eos_token='(EOS)' )
           snake_case__					:      int  			=						self.get_feature_extractor(do_normalize=__lowerCamelCase    ,       padding_value=1.0 )

           snake_case__					:      Any  			=						SpeechaTextProcessor.from_pretrained(
               self.tmpdirname    ,       bos_token='(BOS)'    ,       eos_token='(EOS)'    ,       do_normalize=__lowerCamelCase    ,       padding_value=1.0 )

           self.assertEqual(processor.tokenizer.get_vocab()    ,       tokenizer_add_kwargs.get_vocab() )
           self.assertIsInstance(processor.tokenizer    ,       __lowerCamelCase )

           self.assertEqual(processor.feature_extractor.to_json_string()    ,       feature_extractor_add_kwargs.to_json_string() )
           self.assertIsInstance(processor.feature_extractor    ,       __lowerCamelCase )





     def  _lowerCAmelCase			(    self    :				Any ):
           snake_case__					:      int  			=						self.get_feature_extractor()
           snake_case__					:      Optional[Any]  			=						self.get_tokenizer()

           snake_case__					:      List[str]  			=						SpeechaTextProcessor(tokenizer=__lowerCamelCase    ,       feature_extractor=__lowerCamelCase )

           snake_case__					:      List[str]  			=						floats_list((3, 1000) )

           snake_case__					:      Tuple  			=						feature_extractor(__lowerCamelCase    ,       return_tensors='np' )
           snake_case__					:      str  			=						processor(__lowerCamelCase    ,       return_tensors='np' )

           for key in input_feat_extract.keys():
                 self.assertAlmostEqual(input_feat_extract[key].sum()    ,       input_processor[key].sum()    ,       delta=1E-2 )





     def  _lowerCAmelCase			(    self    :				Dict ):
           snake_case__					:      Optional[int]  			=						self.get_feature_extractor()
           snake_case__					:      Optional[Any]  			=						self.get_tokenizer()

           snake_case__					:      Tuple  			=						SpeechaTextProcessor(tokenizer=__lowerCamelCase    ,       feature_extractor=__lowerCamelCase )

           snake_case__					:      Optional[int]  			=						'This is a test string'

           snake_case__					:      int  			=						processor(text=__lowerCamelCase )

           snake_case__					:      Tuple  			=						tokenizer(__lowerCamelCase )

           for key in encoded_tok.keys():
                 self.assertListEqual(encoded_tok[key]    ,       encoded_processor[key] )





     def  _lowerCAmelCase			(    self    :				int ):
           snake_case__					:      Optional[int]  			=						self.get_feature_extractor()
           snake_case__					:      int  			=						self.get_tokenizer()

           snake_case__					:      str  			=						SpeechaTextProcessor(tokenizer=__lowerCamelCase    ,       feature_extractor=__lowerCamelCase )

           snake_case__					:      str  			=						[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

           snake_case__					:      List[str]  			=						processor.batch_decode(__lowerCamelCase )
           snake_case__					:      Optional[int]  			=						tokenizer.batch_decode(__lowerCamelCase )

           self.assertListEqual(__lowerCamelCase    ,       __lowerCamelCase )





     def  _lowerCAmelCase			(    self    :				str ):
           snake_case__					:      List[str]  			=						self.get_feature_extractor()
           snake_case__					:      int  			=						self.get_tokenizer()

           snake_case__					:      int  			=						SpeechaTextProcessor(tokenizer=__lowerCamelCase    ,       feature_extractor=__lowerCamelCase )

           self.assertListEqual(
               processor.model_input_names    ,       feature_extractor.model_input_names    ,       msg='`processor` and `feature_extractor` model input names do not match'    ,       )
