






from __future__ import annotations

import unittest

from transformers import OpenAIGPTConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers.models.openai.modeling_tf_openai import (
		    TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFOpenAIGPTDoubleHeadsModel,
		    TFOpenAIGPTForSequenceClassification,
		    TFOpenAIGPTLMHeadModel,
		    TFOpenAIGPTModel,
		)



class 						lowerCamelCase   :







		'''simple docstring'''



		def __init__(							self       ,	_UpperCamelCase       ,	)			->							Union[str, Any]:
							UpperCAmelCase_					:   List[Any]   =		parent
							UpperCAmelCase_					:   List[Any]   =		1_3
							UpperCAmelCase_					:   Any   =		7
							UpperCAmelCase_					:   Any   =		True
							UpperCAmelCase_					:   List[Any]   =		True
							UpperCAmelCase_					:   int   =		True
							UpperCAmelCase_					:   Optional[int]   =		True
							UpperCAmelCase_					:   Optional[int]   =		True
							UpperCAmelCase_					:   List[str]   =		9_9
							UpperCAmelCase_					:   str   =		3_2
							UpperCAmelCase_					:   Optional[int]   =		2
							UpperCAmelCase_					:   Union[str, Any]   =		4
							UpperCAmelCase_					:   Any   =		3_7
							UpperCAmelCase_					:   Optional[Any]   =		'gelu'
							UpperCAmelCase_					:   List[str]   =		0.1
							UpperCAmelCase_					:   Any   =		0.1
							UpperCAmelCase_					:   List[Any]   =		5_1_2
							UpperCAmelCase_					:   List[str]   =		1_6
							UpperCAmelCase_					:   Tuple   =		2
							UpperCAmelCase_					:   Optional[int]   =		0.02
							UpperCAmelCase_					:   str   =		3
							UpperCAmelCase_					:   Dict   =		4
							UpperCAmelCase_					:   Optional[Any]   =		None
							UpperCAmelCase_					:   int   =		self.vocab_size - 1



		def       __UpperCAmelCase    (							self     )			->							Dict:
							UpperCAmelCase_					:   Tuple   =		ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size     )

							UpperCAmelCase_					:   str   =		None
							if self.use_input_mask:
												UpperCAmelCase_					:   Tuple   =		random_attention_mask([self.batch_size, self.seq_length]     )

							UpperCAmelCase_					:   str   =		None
							if self.use_token_type_ids:
												UpperCAmelCase_					:   Tuple   =		ids_tensor([self.batch_size, self.seq_length]       ,	self.type_vocab_size     )

							UpperCAmelCase_					:   str   =		None
							if self.use_mc_token_ids:
												UpperCAmelCase_					:   Optional[Any]   =		ids_tensor([self.batch_size, self.num_choices]       ,	self.seq_length     )

							UpperCAmelCase_					:   List[str]   =		None
							UpperCAmelCase_					:   Optional[int]   =		None
							UpperCAmelCase_					:   List[str]   =		None
							if self.use_labels:
												UpperCAmelCase_					:   str   =		ids_tensor([self.batch_size]       ,	self.type_sequence_label_size     )
												UpperCAmelCase_					:   Union[str, Any]   =		ids_tensor([self.batch_size, self.seq_length]       ,	self.num_labels     )
												UpperCAmelCase_					:   Dict   =		ids_tensor([self.batch_size]       ,	self.num_choices     )

							UpperCAmelCase_					:   str   =		OpenAIGPTConfig(
							    vocab_size=self.vocab_size       ,	n_embd=self.hidden_size       ,	n_layer=self.num_hidden_layers       ,	n_head=self.num_attention_heads       ,	n_positions=self.max_position_embeddings       ,	pad_token_id=self.pad_token_id       ,	)

							UpperCAmelCase_					:   Union[str, Any]   =		ids_tensor([self.num_hidden_layers, self.num_attention_heads]       ,	2     )

							return (
							    config,
							    input_ids,
							    input_mask,
							    head_mask,
							    token_type_ids,
							    mc_token_ids,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							)



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	*_UpperCamelCase     )			->							List[Any]:
							UpperCAmelCase_					:   Optional[Any]   =		TFOpenAIGPTModel(config=__a     )
							UpperCAmelCase_					:   Dict   =		{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
							UpperCAmelCase_					:   int   =		model(__a     )

							UpperCAmelCase_					:   int   =		[input_ids, input_mask]
							UpperCAmelCase_					:   List[Any]   =		model(__a     )

							UpperCAmelCase_					:   Tuple   =		model(__a     )

							self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.hidden_size)     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	*_UpperCamelCase     )			->							List[str]:
							UpperCAmelCase_					:   Dict   =		TFOpenAIGPTLMHeadModel(config=__a     )
							UpperCAmelCase_					:   Tuple   =		{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
							UpperCAmelCase_					:   int   =		model(__a     )
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	*_UpperCamelCase     )			->							str:
							UpperCAmelCase_					:   Optional[Any]   =		TFOpenAIGPTDoubleHeadsModel(config=__a     )

							UpperCAmelCase_					:   List[Any]   =		tf.tile(tf.expand_dims(__a       ,	1     )       ,	(1, self.num_choices, 1)     )
							UpperCAmelCase_					:   List[Any]   =		tf.tile(tf.expand_dims(__a       ,	1     )       ,	(1, self.num_choices, 1)     )
							UpperCAmelCase_					:   Union[str, Any]   =		tf.tile(tf.expand_dims(__a       ,	1     )       ,	(1, self.num_choices, 1)     )

							UpperCAmelCase_					:   Union[str, Any]   =		{
							    'input_ids': multiple_choice_inputs_ids,
							    'mc_token_ids': mc_token_ids,
							    'attention_mask': multiple_choice_input_mask,
							    'token_type_ids': multiple_choice_token_type_ids,
							}
							UpperCAmelCase_					:   int   =		model(__a     )
							self.parent.assertEqual(
							    result.logits.shape       ,	(self.batch_size, self.num_choices, self.seq_length, self.vocab_size)     )
							self.parent.assertEqual(result.mc_logits.shape       ,	(self.batch_size, self.num_choices)     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	*_UpperCamelCase     )			->							Tuple:
							UpperCAmelCase_					:   Tuple   =		self.num_labels
							UpperCAmelCase_					:   int   =		ids_tensor([self.batch_size]       ,	self.type_sequence_label_size     )
							UpperCAmelCase_					:   List[Any]   =		{
							    'input_ids': input_ids,
							    'attention_mask': input_mask,
							    'token_type_ids': token_type_ids,
							    'labels': sequence_labels,
							}
							UpperCAmelCase_					:   Optional[int]   =		TFOpenAIGPTForSequenceClassification(__a     )
							UpperCAmelCase_					:   List[str]   =		model(__a     )
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.num_labels)     )



		def       __UpperCAmelCase    (							self     )			->							List[Any]:
							UpperCAmelCase_					:   List[str]   =		self.prepare_config_and_inputs()

							(
							    UpperCAmelCase_       
							)					:   Optional[int]   =		config_and_inputs

							UpperCAmelCase_					:   Union[str, Any]   =		{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
							return config, inputs_dict





@require_tf
class 						lowerCamelCase   (lowerCAmelCase_				,						lowerCAmelCase_				,						unittest.TestCase    ):







		'''simple docstring'''



		_snake_case					:					Union[str, Any] 		=						(
		    (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TFOpenAIGPTForSequenceClassification)
		    if is_tf_available()
		    else ()
		)
		_snake_case					:					Dict 		=						(
		    (TFOpenAIGPTLMHeadModel,) if is_tf_available() else ()
		)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
		_snake_case					:					List[str] 		=						(
		    {
		        '''feature-extraction''': TFOpenAIGPTModel,
		        '''text-classification''': TFOpenAIGPTForSequenceClassification,
		        '''text-generation''': TFOpenAIGPTLMHeadModel,
		        '''zero-shot''': TFOpenAIGPTForSequenceClassification,
		    }
		    if is_tf_available()
		    else {}
		)
		_snake_case					:					Any 		=						False
		_snake_case					:					Dict 		=						False



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase     )			->							Any:





							if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
												# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
												# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
												# tiny config could not be created.
												return True

							return False



		def       __UpperCAmelCase    (							self     )			->							Dict:
							UpperCAmelCase_					:   Optional[Any]   =		TFOpenAIGPTModelTester(self     )
							UpperCAmelCase_					:   Optional[Any]   =		ConfigTester(self       ,	config_class=__a       ,	n_embd=3_7     )



		def       __UpperCAmelCase    (							self     )			->							List[Any]:
							self.config_tester.run_common_tests()



		def       __UpperCAmelCase    (							self     )			->							Union[str, Any]:
							UpperCAmelCase_					:   Union[str, Any]   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_openai_gpt_model(*__a     )



		def       __UpperCAmelCase    (							self     )			->							List[Any]:
							UpperCAmelCase_					:   Tuple   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_openai_gpt_lm_head(*__a     )



		def       __UpperCAmelCase    (							self     )			->							List[Any]:
							UpperCAmelCase_					:   List[str]   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_openai_gpt_double_head(*__a     )



		def       __UpperCAmelCase    (							self     )			->							List[Any]:
							UpperCAmelCase_					:   str   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*__a     )



		@slow
		def       __UpperCAmelCase    (							self     )			->							Union[str, Any]:





							for model_name in TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												UpperCAmelCase_					:   List[str]   =		TFOpenAIGPTModel.from_pretrained(__a     )
												self.assertIsNotNone(__a     )





@require_tf
class 						lowerCamelCase   (unittest.TestCase    ):







		'''simple docstring'''



		@slow
		def       __UpperCAmelCase    (							self     )			->							Optional[int]:
							UpperCAmelCase_					:   Optional[int]   =		TFOpenAIGPTLMHeadModel.from_pretrained('openai-gpt'     )
							UpperCAmelCase_					:   str   =		tf.convert_to_tensor([[4_8_1, 4_7_3_5, 5_4_4]]       ,	dtype=tf.intaa     )  # the president is
							UpperCAmelCase_					:   Dict   =		[
							    4_8_1,
							    4_7_3_5,
							    5_4_4,
							    2_4_6,
							    9_6_3,
							    8_7_0,
							    7_6_2,
							    2_3_9,
							    2_4_4,
							    4_0_4_7_7,
							    2_4_4,
							    2_4_9,
							    7_1_9,
							    8_8_1,
							    4_8_7,
							    5_4_4,
							    2_4_0,
							    2_4_4,
							    6_0_3,
							    4_8_1,
							]  # the president is a very good man. " \n " i\'m sure he is, " said the

							UpperCAmelCase_					:   List[Any]   =		model.generate(__a       ,	do_sample=__a     )
							self.assertListEqual(output_ids[0].numpy().tolist()       ,	__a     )














import os
import unittest

from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class 						lowerCamelCase   (_snake_case				,						unittest.TestCase    ):







		'''simple docstring'''



		_snake_case					:					Tuple 		=						BertweetTokenizer
		_snake_case					:					List[str] 		=						False



		def       __UpperCAmelCase    (							self     )			->							Optional[int]:
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							UpperCAmelCase_					:   Optional[int]   =		['I', 'm', 'V@@', 'R@@', 'r', 'e@@']
							UpperCAmelCase_					:   Dict   =		dict(zip(_UpperCamelCase       ,	range(len(_UpperCamelCase     )     )     )     )
							UpperCAmelCase_					:   List[Any]   =		['#version: 0.2', 'a m</w>']
							UpperCAmelCase_					:   Union[str, Any]   =		{'unk_token': '<unk>'}

							UpperCAmelCase_					:   Optional[Any]   =		os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES['vocab_file']     )
							UpperCAmelCase_					:   List[str]   =		os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES['merges_file']     )
							with open(self.vocab_file       ,	'w'       ,	encoding='utf-8'     ) as fp:
												for token in vocab_tokens:
																	fp.write(f"{token} {vocab_tokens[token]}\n"     )
							with open(self.merges_file       ,	'w'       ,	encoding='utf-8'     ) as fp:
												fp.write('\n'.join(_UpperCamelCase     )     )



		def       __UpperCAmelCase    (							self       ,	**_UpperCamelCase     )			->							List[Any]:
							kwargs.update(self.special_tokens_map     )
							return BertweetTokenizer.from_pretrained(self.tmpdirname       ,	**_UpperCamelCase     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase     )			->							List[Any]:
							UpperCAmelCase_					:   Any   =		'I am VinAI Research'
							UpperCAmelCase_					:   Any   =		'I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'
							return input_text, output_text



		def       __UpperCAmelCase    (							self     )			->							Tuple:
							UpperCAmelCase_					:   Union[str, Any]   =		BertweetTokenizer(self.vocab_file       ,	self.merges_file       ,	**self.special_tokens_map     )
							UpperCAmelCase_					:   Optional[Any]   =		'I am VinAI Research'
							UpperCAmelCase_					:   str   =		'I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'.split()
							UpperCAmelCase_					:   Any   =		tokenizer.tokenize(_UpperCamelCase     )
							self.assertListEqual(_UpperCamelCase       ,	_UpperCamelCase     )

							UpperCAmelCase_					:   Optional[int]   =		tokens + [tokenizer.unk_token]

							UpperCAmelCase_					:   List[str]   =		[4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase     )       ,	_UpperCamelCase     )






