



import argparse
import json
import logging
import os
import sys
from unittest import skip
from unittest.mock import patch

import tensorflow as tf

from transformers.testing_utils import TestCasePlus, get_gpu_count, slow


A__						:						Union[str, Any]          =      [
    os.path.join(os.path.dirname(__file__), dirname)
    for dirname in [
        '''text-generation''',
        '''text-classification''',
        '''token-classification''',
        '''language-modeling''',
        '''multiple-choice''',
        '''question-answering''',
        '''summarization''',
        '''translation''',
        '''image-classification''',
    ]
]
sys.path.extend(SRC_DIRS)


if SRC_DIRS is not None:
						import run_clm
						import run_image_classification
						import run_mlm
						import run_ner
						import run_qa as run_squad
						import run_summarization
						import run_swag
						import run_text_classification
						import run_translation


logging.basicConfig(level=logging.DEBUG)

A__						:						List[Any]          =      logging.getLogger()


def     UpperCamelCase(						):
		lowerCAmelCase_    :  List[str]             =				argparse.ArgumentParser()
		parser.add_argument('''-f'''				)
		lowerCAmelCase_    :  Any             =				parser.parse_args()
		return args.f


def     UpperCamelCase(						__UpperCamelCase							:	Dict				):
		lowerCAmelCase_    :  Optional[Any]             =				{}
		lowerCAmelCase_    :  List[Any]             =				os.path.join(__UpperCamelCase  ,'''all_results.json'''				)
		if os.path.exists(__UpperCamelCase				):
				with open(__UpperCamelCase  ,'''r'''				) as f:
						lowerCAmelCase_    :  Tuple             =				json.load(__UpperCamelCase				)
		else:
				raise ValueError(f"""can't find {path}"""				)
		return results


def     UpperCamelCase(						):
		return bool(tf.config.list_physical_devices('''GPU'''				)				)


A__						:						Optional[Any]          =      logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)





class        __snake_case		(  UpperCamelCase_		):




							@skip('''Skipping until shape inference for to_tf_dataset PR is merged.''')
							def      UpperCAmelCase__    ( self  : Optional[Any]):
									lowerCAmelCase_    :  List[Any]             =				self.get_auto_remove_tmp_dir()
									lowerCAmelCase_    :  Optional[Any]             =				F"""
            run_text_classification.py
            --model_name_or_path distilbert-base-uncased
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
            --do_train
            --do_eval
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --learning_rate=1e-4
            --max_steps=10
            --warmup_steps=2
            --seed=42
            --max_seq_length=128
            """.split()

									if is_cuda_available():
											testargs.append('''--fp16''')

									with patch.object(A_       ,       '''argv'''       ,       A_):
											run_text_classification.main()
											# Reset the mixed precision policy so we don't break other tests
											tf.keras.mixed_precision.set_global_policy('''float32''')
											lowerCAmelCase_    :  Dict             =				get_results(A_)
											self.assertGreaterEqual(result['''eval_accuracy''']       ,       0.75)




							def      UpperCAmelCase__    ( self  : int):
									lowerCAmelCase_    :  List[str]             =				self.get_auto_remove_tmp_dir()
									lowerCAmelCase_    :  Any             =				F"""
            run_clm.py
            --model_name_or_path distilgpt2
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --do_train
            --do_eval
            --block_size 128
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --num_train_epochs 2
            --output_dir {tmp_dir}
            --overwrite_output_dir
            """.split()

									if len(tf.config.list_physical_devices('''GPU''')) > 1:
											# Skipping because there are not enough batches to train the model + would need a drop_last to work.
											return

									with patch.object(A_       ,       '''argv'''       ,       A_):
											run_clm.main()
											lowerCAmelCase_    :  Optional[int]             =				get_results(A_)
											self.assertLess(result['''eval_perplexity''']       ,       1_0_0)




							def      UpperCAmelCase__    ( self  : List[Any]):
									lowerCAmelCase_    :  Any             =				self.get_auto_remove_tmp_dir()
									lowerCAmelCase_    :  Tuple             =				F"""
            run_mlm.py
            --model_name_or_path distilroberta-base
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --max_seq_length 64
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --do_train
            --do_eval
            --prediction_loss_only
            --num_train_epochs=1
            --learning_rate=1e-4
        """.split()

									with patch.object(A_       ,       '''argv'''       ,       A_):
											run_mlm.main()
											lowerCAmelCase_    :  List[str]             =				get_results(A_)
											self.assertLess(result['''eval_perplexity''']       ,       4_2)




							def      UpperCAmelCase__    ( self  : Optional[int]):
									# with so little data distributed training needs more epochs to get the score on par with 0/1 gpu
									lowerCAmelCase_    :  List[str]             =				7 if get_gpu_count() > 1 else 2

									lowerCAmelCase_    :  Tuple             =				self.get_auto_remove_tmp_dir()
									lowerCAmelCase_    :  List[str]             =				F"""
            run_ner.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/conll/sample.json
            --validation_file tests/fixtures/tests_samples/conll/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --do_train
            --do_eval
            --warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=2
            --num_train_epochs={epochs}
            --seed 7
        """.split()

									with patch.object(A_       ,       '''argv'''       ,       A_):
											run_ner.main()
											lowerCAmelCase_    :  int             =				get_results(A_)
											self.assertGreaterEqual(result['''accuracy''']       ,       0.75)




							def      UpperCAmelCase__    ( self  : List[Any]):
									lowerCAmelCase_    :  Union[str, Any]             =				self.get_auto_remove_tmp_dir()
									lowerCAmelCase_    :  List[Any]             =				F"""
            run_qa.py
            --model_name_or_path bert-base-uncased
            --version_2_with_negative
            --train_file tests/fixtures/tests_samples/SQUAD/sample.json
            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_steps=10
            --warmup_steps=2
            --do_train
            --do_eval
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
        """.split()

									with patch.object(A_       ,       '''argv'''       ,       A_):
											run_squad.main()
											lowerCAmelCase_    :  Optional[int]             =				get_results(A_)
											self.assertGreaterEqual(result['''f1''']       ,       3_0)
											self.assertGreaterEqual(result['''exact''']       ,       3_0)




							def      UpperCAmelCase__    ( self  : Union[str, Any]):
									lowerCAmelCase_    :  Any             =				self.get_auto_remove_tmp_dir()
									lowerCAmelCase_    :  int             =				F"""
            run_swag.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/swag/sample.json
            --validation_file tests/fixtures/tests_samples/swag/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_steps=20
            --warmup_steps=2
            --do_train
            --do_eval
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
        """.split()

									with patch.object(A_       ,       '''argv'''       ,       A_):
											run_swag.main()
											lowerCAmelCase_    :  List[str]             =				get_results(A_)
											self.assertGreaterEqual(result['''val_accuracy''']       ,       0.8)




							@slow
							def      UpperCAmelCase__    ( self  : List[Any]):
									lowerCAmelCase_    :  Optional[int]             =				self.get_auto_remove_tmp_dir()
									lowerCAmelCase_    :  Dict             =				F"""
            run_summarization.py
            --model_name_or_path t5-small
            --train_file tests/fixtures/tests_samples/xsum/sample.json
            --validation_file tests/fixtures/tests_samples/xsum/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_steps=50
            --warmup_steps=8
            --do_train
            --do_eval
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
        """.split()

									with patch.object(A_       ,       '''argv'''       ,       A_):
											run_summarization.main()
											lowerCAmelCase_    :  str             =				get_results(A_)
											self.assertGreaterEqual(result['''rouge1''']       ,       1_0)
											self.assertGreaterEqual(result['''rouge2''']       ,       2)
											self.assertGreaterEqual(result['''rougeL''']       ,       7)
											self.assertGreaterEqual(result['''rougeLsum''']       ,       7)




							@slow
							def      UpperCAmelCase__    ( self  : str):
									lowerCAmelCase_    :  Dict             =				self.get_auto_remove_tmp_dir()
									lowerCAmelCase_    :  Optional[int]             =				F"""
            run_translation.py
            --model_name_or_path Rocketknight1/student_marian_en_ro_6_1
            --source_lang en
            --target_lang ro
            --train_file tests/fixtures/tests_samples/wmt16/sample.json
            --validation_file tests/fixtures/tests_samples/wmt16/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --warmup_steps=8
            --do_train
            --do_eval
            --learning_rate=3e-3
            --num_train_epochs 12
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --source_lang en_XX
            --target_lang ro_RO
        """.split()

									with patch.object(A_       ,       '''argv'''       ,       A_):
											run_translation.main()
											lowerCAmelCase_    :  Optional[Any]             =				get_results(A_)
											self.assertGreaterEqual(result['''bleu''']       ,       3_0)







							def      UpperCAmelCase__    ( self  : List[str]):
									lowerCAmelCase_    :  Tuple             =				self.get_auto_remove_tmp_dir()
									lowerCAmelCase_    :  Tuple             =				F"""
            run_image_classification.py
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --model_name_or_path microsoft/resnet-18
            --do_train
            --do_eval
            --learning_rate 1e-4
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --dataloader_num_workers 16
            --num_train_epochs 2
            --train_val_split 0.1
            --seed 42
            --ignore_mismatched_sizes True
            """.split()

									with patch.object(A_       ,       '''argv'''       ,       A_):
											run_image_classification.main()
											lowerCAmelCase_    :  Dict             =				get_results(A_)
											self.assertGreaterEqual(result['''accuracy''']       ,       0.7)










import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
						import torch

if is_vision_available():
						from PIL import Image

						from transformers import DeiTImageProcessor



class        __snake_case		(  unittest.TestCase		):




							def __init__( self  : Union[str, Any]       ,       A_  : Optional[Any]       ,       A_  : Optional[Any]=7       ,       A_  : str=3       ,       A_  : Optional[Any]=1_8       ,       A_  : Union[str, Any]=3_0       ,       A_  : str=4_0_0       ,       A_  : List[str]=True       ,       A_  : str=None       ,       A_  : str=True       ,       A_  : List[Any]=None       ,       A_  : List[Any]=True       ,       A_  : Optional[Any]=[0.5, 0.5, 0.5]       ,       A_  : Optional[int]=[0.5, 0.5, 0.5]       ,       ):
									lowerCAmelCase_    :  List[str]             =				size if size is not None else {'''height''': 2_0, '''width''': 2_0}
									lowerCAmelCase_    :  List[Any]             =				crop_size if crop_size is not None else {'''height''': 1_8, '''width''': 1_8}

									lowerCAmelCase_    :  str             =				parent
									lowerCAmelCase_    :  Union[str, Any]             =				batch_size
									lowerCAmelCase_    :  Optional[Any]             =				num_channels
									lowerCAmelCase_    :  List[str]             =				image_size
									lowerCAmelCase_    :  List[str]             =				min_resolution
									lowerCAmelCase_    :  Union[str, Any]             =				max_resolution
									lowerCAmelCase_    :  Optional[int]             =				do_resize
									lowerCAmelCase_    :  Dict             =				size
									lowerCAmelCase_    :  Dict             =				do_center_crop
									lowerCAmelCase_    :  str             =				crop_size
									lowerCAmelCase_    :  List[Any]             =				do_normalize
									lowerCAmelCase_    :  str             =				image_mean
									lowerCAmelCase_    :  Dict             =				image_std







							def      UpperCAmelCase__    ( self  : int):
									return {
									    "do_resize": self.do_resize,
									    "size": self.size,
									    "do_center_crop": self.do_center_crop,
									    "crop_size": self.crop_size,
									    "do_normalize": self.do_normalize,
									    "image_mean": self.image_mean,
									    "image_std": self.image_std,
									}





@require_torch
@require_vision
class        __snake_case		(  UpperCamelCase_		,unittest.TestCase		):
							_a       =				DeiTImageProcessor if is_vision_available() else None
							_a       =				True




							def      UpperCAmelCase__    ( self  : Tuple):
									lowerCAmelCase_    :  int             =				DeiTImageProcessingTester(self)




							@property
							def      UpperCAmelCase__    ( self  : Optional[Any]):
									return self.image_processor_tester.prepare_image_processor_dict()




							def      UpperCAmelCase__    ( self  : str):
									lowerCAmelCase_    :  List[Any]             =				self.image_processing_class(**self.image_processor_dict)
									self.assertTrue(hasattr(A_       ,       '''do_resize'''))
									self.assertTrue(hasattr(A_       ,       '''size'''))
									self.assertTrue(hasattr(A_       ,       '''do_center_crop'''))
									self.assertTrue(hasattr(A_       ,       '''center_crop'''))
									self.assertTrue(hasattr(A_       ,       '''do_normalize'''))
									self.assertTrue(hasattr(A_       ,       '''image_mean'''))
									self.assertTrue(hasattr(A_       ,       '''image_std'''))




							def      UpperCAmelCase__    ( self  : Union[str, Any]):
									lowerCAmelCase_    :  Tuple             =				self.image_processing_class.from_dict(self.image_processor_dict)
									self.assertEqual(image_processor.size       ,       {'''height''': 2_0, '''width''': 2_0})
									self.assertEqual(image_processor.crop_size       ,       {'''height''': 1_8, '''width''': 1_8})

									lowerCAmelCase_    :  Any             =				self.image_processing_class.from_dict(self.image_processor_dict       ,       size=4_2       ,       crop_size=8_4)
									self.assertEqual(image_processor.size       ,       {'''height''': 4_2, '''width''': 4_2})
									self.assertEqual(image_processor.crop_size       ,       {'''height''': 8_4, '''width''': 8_4})




							def      UpperCAmelCase__    ( self  : Union[str, Any]):
									pass




							def      UpperCAmelCase__    ( self  : Any):
									# Initialize image_processing
									lowerCAmelCase_    :  List[str]             =				self.image_processing_class(**self.image_processor_dict)
									# create random PIL images
									lowerCAmelCase_    :  Any             =				prepare_image_inputs(self.image_processor_tester       ,       equal_resolution=A_)
									for image in image_inputs:
											self.assertIsInstance(A_       ,       Image.Image)

									# Test not batched input
									lowerCAmelCase_    :  Union[str, Any]             =				image_processing(image_inputs[0]       ,       return_tensors='''pt''').pixel_values
									self.assertEqual(
									    encoded_images.shape       ,       (
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['''height'''],
									        self.image_processor_tester.crop_size['''width'''],
									    )       ,       )

									# Test batched
									lowerCAmelCase_    :  Dict             =				image_processing(A_       ,       return_tensors='''pt''').pixel_values
									self.assertEqual(
									    encoded_images.shape       ,       (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['''height'''],
									        self.image_processor_tester.crop_size['''width'''],
									    )       ,       )




							def      UpperCAmelCase__    ( self  : List[str]):
									# Initialize image_processing
									lowerCAmelCase_    :  Union[str, Any]             =				self.image_processing_class(**self.image_processor_dict)
									# create random numpy tensors
									lowerCAmelCase_    :  Union[str, Any]             =				prepare_image_inputs(self.image_processor_tester       ,       equal_resolution=A_       ,       numpify=A_)
									for image in image_inputs:
											self.assertIsInstance(A_       ,       np.ndarray)

									# Test not batched input
									lowerCAmelCase_    :  Optional[Any]             =				image_processing(image_inputs[0]       ,       return_tensors='''pt''').pixel_values
									self.assertEqual(
									    encoded_images.shape       ,       (
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['''height'''],
									        self.image_processor_tester.crop_size['''width'''],
									    )       ,       )

									# Test batched
									lowerCAmelCase_    :  str             =				image_processing(A_       ,       return_tensors='''pt''').pixel_values
									self.assertEqual(
									    encoded_images.shape       ,       (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['''height'''],
									        self.image_processor_tester.crop_size['''width'''],
									    )       ,       )







							def      UpperCAmelCase__    ( self  : Any):
									# Initialize image_processing
									lowerCAmelCase_    :  str             =				self.image_processing_class(**self.image_processor_dict)
									# create random PyTorch tensors
									lowerCAmelCase_    :  Union[str, Any]             =				prepare_image_inputs(self.image_processor_tester       ,       equal_resolution=A_       ,       torchify=A_)
									for image in image_inputs:
											self.assertIsInstance(A_       ,       torch.Tensor)

									# Test not batched input
									lowerCAmelCase_    :  Dict             =				image_processing(image_inputs[0]       ,       return_tensors='''pt''').pixel_values
									self.assertEqual(
									    encoded_images.shape       ,       (
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['''height'''],
									        self.image_processor_tester.crop_size['''width'''],
									    )       ,       )

									# Test batched
									lowerCAmelCase_    :  Tuple             =				image_processing(A_       ,       return_tensors='''pt''').pixel_values
									self.assertEqual(
									    encoded_images.shape       ,       (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size['''height'''],
									        self.image_processor_tester.crop_size['''width'''],
									    )       ,       )





