
'''simple docstring'''







from copy import deepcopy
from unittest.case import TestCase

import pytest

from datasets.arrow_dataset import Dataset
from datasets.features import Audio, ClassLabel, Features, Image, Sequence, Value
from datasets.info import DatasetInfo
from datasets.tasks import (
    AudioClassification,
    AutomaticSpeechRecognition,
    ImageClassification,
    LanguageModeling,
    QuestionAnsweringExtractive,
    Summarization,
    TextClassification,
    task_template_from_dict,
)
from datasets.utils.py_utils import asdict


_lowercase				:						Any												= {
    "id": "5733be284776f41900661182",
    "title": "University_of_Notre_Dame",
    "context": "Architecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.",
    "question": "To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?",
    "answers": {"text": ["Saint Bernadette Soubirous"], "answer_start": [5_1_5]},
}
@pytest.mark.parametrize(
    '''task_cls'''   ,   [
        AudioClassification,
        AutomaticSpeechRecognition,
        ImageClassification,
        LanguageModeling,
        QuestionAnsweringExtractive,
        Summarization,
        TextClassification,
    ]   ,   )
def       snake_case_	(       __SCREAMING_SNAKE_CASE      :				Any							):



						"""simple docstring"""





						lowercase_			:  List[Any]									= task_cls()
						lowercase_			:  Optional[int]									= asdict(__SCREAMING_SNAKE_CASE							)
						lowercase_			:  str									= task_template_from_dict(__SCREAMING_SNAKE_CASE							)
						assert task == reloaded






class 				lowerCAmelCase__     :






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Optional[int]									= LanguageModeling(text_column='''input_text'''	)
											assert {"input_text": "text"} == task.column_mapping






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  List[Any]									= Features({'''text''': Value('''string'''	)}	)
											lowercase_			:  Optional[int]									= {'''text_column''': '''input_text'''}
											lowercase_			:  List[Any]									= LanguageModeling.from_dict(__SCREAMING_SNAKE_CASE	)
											assert "language-modeling" == task.task
											assert input_schema == task.input_schema







class 				lowerCAmelCase__     (						lowerCamelCase_  ):






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Dict									= sorted(['''pos''', '''neg''']	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  str									= TextClassification(text_column='''input_text'''       ,							label_column='''input_label'''	)
											self.assertDictEqual({'''input_text''': '''text''', '''input_label''': '''labels'''}       ,							task.column_mapping	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Any									= Features({'''text''': Value('''string'''	)}	)
											# Labels are cast to tuple during `TextClassification.__post_init__`, so we do the same here
											lowercase_			:  Union[str, Any]									= Features({'''labels''': ClassLabel}	)
											lowercase_			:  str									= {'''text_column''': '''input_text''', '''label_column''': '''input_labels'''}
											lowercase_			:  Any									= TextClassification.from_dict(__SCREAMING_SNAKE_CASE	)
											self.assertEqual('''text-classification'''       ,							task.task	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.input_schema	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.label_schema	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  str									= TextClassification(text_column='''input_text'''       ,							label_column='''input_label'''	)
											self.assertEqual(task.label_schema['''labels''']       ,							__SCREAMING_SNAKE_CASE	)
											lowercase_			:  str									= task.align_with_features(Features({'''input_label''': ClassLabel(names=self.labels	)}	)	)
											self.assertEqual(task.label_schema['''labels''']       ,							ClassLabel(names=self.labels	)	)







class 				lowerCAmelCase__     (						lowerCamelCase_  ):






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  int									= QuestionAnsweringExtractive(
											    context_column='''input_context'''       ,							question_column='''input_question'''       ,							answers_column='''input_answers'''	)
											self.assertDictEqual(
											    {'''input_context''': '''context''', '''input_question''': '''question''', '''input_answers''': '''answers'''}       ,							task.column_mapping	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Union[str, Any]									= Features({'''question''': Value('''string'''	), '''context''': Value('''string'''	)}	)
											lowercase_			:  Union[str, Any]									= Features(
											    {
											        '''answers''': Sequence(
											            {
											                '''text''': Value('''string'''	),
											                '''answer_start''': Value('''int32'''	),
											            }	)
											    }	)
											lowercase_			:  List[Any]									= {
											    '''context_column''': '''input_input_context''',
											    '''question_column''': '''input_question''',
											    '''answers_column''': '''input_answers''',
											}
											lowercase_			:  Optional[int]									= QuestionAnsweringExtractive.from_dict(__SCREAMING_SNAKE_CASE	)
											self.assertEqual('''question-answering-extractive'''       ,							task.task	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.input_schema	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.label_schema	)







class 				lowerCAmelCase__     (						lowerCamelCase_  ):






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Union[str, Any]									= Summarization(text_column='''input_text'''       ,							summary_column='''input_summary'''	)
											self.assertDictEqual({'''input_text''': '''text''', '''input_summary''': '''summary'''}       ,							task.column_mapping	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  List[Any]									= Features({'''text''': Value('''string'''	)}	)
											lowercase_			:  List[Any]									= Features({'''summary''': Value('''string'''	)}	)
											lowercase_			:  Optional[int]									= {'''text_column''': '''input_text''', '''summary_column''': '''input_summary'''}
											lowercase_			:  List[str]									= Summarization.from_dict(__SCREAMING_SNAKE_CASE	)
											self.assertEqual('''summarization'''       ,							task.task	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.input_schema	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.label_schema	)







class 				lowerCAmelCase__     (						lowerCamelCase_  ):






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  str									= AutomaticSpeechRecognition(audio_column='''input_audio'''       ,							transcription_column='''input_transcription'''	)
											self.assertDictEqual({'''input_audio''': '''audio''', '''input_transcription''': '''transcription'''}       ,							task.column_mapping	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Optional[int]									= Features({'''audio''': Audio()}	)
											lowercase_			:  Any									= Features({'''transcription''': Value('''string'''	)}	)
											lowercase_			:  Union[str, Any]									= {
											    '''audio_column''': '''input_audio''',
											    '''transcription_column''': '''input_transcription''',
											}
											lowercase_			:  Tuple									= AutomaticSpeechRecognition.from_dict(__SCREAMING_SNAKE_CASE	)
											self.assertEqual('''automatic-speech-recognition'''       ,							task.task	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.input_schema	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.label_schema	)







class 				lowerCAmelCase__     (						lowerCamelCase_  ):






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Optional[Any]									= sorted(['''pos''', '''neg''']	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  str									= AudioClassification(audio_column='''input_audio'''       ,							label_column='''input_label'''	)
											self.assertDictEqual({'''input_audio''': '''audio''', '''input_label''': '''labels'''}       ,							task.column_mapping	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Tuple									= Features({'''audio''': Audio()}	)
											lowercase_			:  Tuple									= Features({'''labels''': ClassLabel}	)
											lowercase_			:  List[str]									= {
											    '''audio_column''': '''input_image''',
											    '''label_column''': '''input_label''',
											}
											lowercase_			:  Union[str, Any]									= AudioClassification.from_dict(__SCREAMING_SNAKE_CASE	)
											self.assertEqual('''audio-classification'''       ,							task.task	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.input_schema	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.label_schema	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  str									= AudioClassification(audio_column='''input_audio'''       ,							label_column='''input_label'''	)
											self.assertEqual(task.label_schema['''labels''']       ,							__SCREAMING_SNAKE_CASE	)
											lowercase_			:  List[str]									= task.align_with_features(Features({'''input_label''': ClassLabel(names=self.labels	)}	)	)
											self.assertEqual(task.label_schema['''labels''']       ,							ClassLabel(names=self.labels	)	)







class 				lowerCAmelCase__     (						lowerCamelCase_  ):






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Any									= sorted(['''pos''', '''neg''']	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  List[str]									= ImageClassification(image_column='''input_image'''       ,							label_column='''input_label'''	)
											self.assertDictEqual({'''input_image''': '''image''', '''input_label''': '''labels'''}       ,							task.column_mapping	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Optional[Any]									= Features({'''image''': Image()}	)
											lowercase_			:  int									= Features({'''labels''': ClassLabel}	)
											lowercase_			:  int									= {
											    '''image_column''': '''input_image''',
											    '''label_column''': '''input_label''',
											}
											lowercase_			:  Any									= ImageClassification.from_dict(__SCREAMING_SNAKE_CASE	)
											self.assertEqual('''image-classification'''       ,							task.task	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.input_schema	)
											self.assertEqual(__SCREAMING_SNAKE_CASE       ,							task.label_schema	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Union[str, Any]									= ImageClassification(image_column='''input_image'''       ,							label_column='''input_label'''	)
											self.assertEqual(task.label_schema['''labels''']       ,							__SCREAMING_SNAKE_CASE	)
											lowercase_			:  int									= task.align_with_features(Features({'''input_label''': ClassLabel(names=self.labels	)}	)	)
											self.assertEqual(task.label_schema['''labels''']       ,							ClassLabel(names=self.labels	)	)







class 				lowerCAmelCase__     (						lowerCamelCase_  ):






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Union[str, Any]									= DatasetInfo(task_templates=QuestionAnsweringExtractive()	)
											lowercase_			:  Any									= Dataset.from_dict({k: [v] for k, v in SAMPLE_QUESTION_ANSWERING_EXTRACTIVE.items()}       ,							info=__SCREAMING_SNAKE_CASE	)
											assert isinstance(dataset.info.task_templates       ,							__SCREAMING_SNAKE_CASE	)
											assert len(dataset.info.task_templates	) == 1

											def keep_task(__SCREAMING_SNAKE_CASE	):
																	return x

											def dont_keep_task(__SCREAMING_SNAKE_CASE	):
																	lowercase_			:  List[Any]									= deepcopy(__SCREAMING_SNAKE_CASE	)
																	lowercase_			:  Optional[int]									= 0
																	return out

											lowercase_			:  int									= dataset.map(__SCREAMING_SNAKE_CASE	)
											assert mapped_dataset.info.task_templates == dataset.info.task_templates
											# reload from cache
											lowercase_			:  Union[str, Any]									= dataset.map(__SCREAMING_SNAKE_CASE	)
											assert mapped_dataset.info.task_templates == dataset.info.task_templates

											lowercase_			:  Union[str, Any]									= dataset.map(__SCREAMING_SNAKE_CASE	)
											assert mapped_dataset.info.task_templates == []
											# reload from cache
											lowercase_			:  str									= dataset.map(__SCREAMING_SNAKE_CASE	)
											assert mapped_dataset.info.task_templates == []






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  str									= Features({'''text''': Value('''string'''	), '''label''': ClassLabel(names=('''pos''', '''neg''')	)}	)
											lowercase_			:  Tuple									= TextClassification(text_column='''text'''       ,							label_column='''label'''	)
											lowercase_			:  Optional[int]									= DatasetInfo(features=__SCREAMING_SNAKE_CASE       ,							task_templates=__SCREAMING_SNAKE_CASE	)
											lowercase_			:  Tuple									= Dataset.from_dict({'''text''': ['''A sentence.'''], '''label''': ['''pos''']}       ,							info=__SCREAMING_SNAKE_CASE	)

											def process(__SCREAMING_SNAKE_CASE	):
																	return example

											lowercase_			:  Optional[Any]									= dataset.remove_columns('''label'''	)
											lowercase_			:  Union[str, Any]									= modified_dataset.map(__SCREAMING_SNAKE_CASE	)
											assert mapped_dataset.info.task_templates == []





'''simple docstring'''







from typing import List

from ...processing_utils import ProcessorMixin
from ...utils import is_torch_available


if is_torch_available():
							import torch






class 				lowerCAmelCase__     (						lowerCamelCase_  ):
					lowerCAmelCase_       =  ['''image_processor''', '''tokenizer''']
					lowerCAmelCase_       =  '''OneFormerImageProcessor'''
					lowerCAmelCase_       =  ('''CLIPTokenizer''', '''CLIPTokenizerFast''')






					def __init__(     self       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE = 77       ,							__SCREAMING_SNAKE_CASE = 77       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											if image_processor is None:
																	raise ValueError('''You need to specify an `image_processor`.'''	)
											if tokenizer is None:
																	raise ValueError('''You need to specify a `tokenizer`.'''	)

											lowercase_			:  Dict									= max_seq_length
											lowercase_			:  Dict									= task_seq_length

											super().__init__(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE=77	):


											"""simple docstring"""





											if text_list is None:
																	raise ValueError('''tokens cannot be None.'''	)

											lowercase_			:  str									= self.tokenizer(__SCREAMING_SNAKE_CASE       ,							padding='''max_length'''       ,							max_length=__SCREAMING_SNAKE_CASE       ,							truncation=__SCREAMING_SNAKE_CASE	)

											lowercase_		,				lowercase_			:  List[str]									= tokens['''attention_mask'''], tokens['''input_ids''']

											lowercase_			:  List[str]									= []
											for attn_mask, input_id in zip(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):
																	lowercase_			:  Optional[Any]									= torch.tensor(__SCREAMING_SNAKE_CASE	) * torch.tensor(__SCREAMING_SNAKE_CASE	)
																	token_inputs.append(token.unsqueeze(0	)	)

											lowercase_			:  str									= torch.cat(__SCREAMING_SNAKE_CASE       ,							dim=0	)
											return token_inputs






					def __call__(     self       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE=None       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											if task_inputs is None:
																	raise ValueError('''You have to specify the task_input. Found None.'''	)
											elif images is None:
																	raise ValueError('''You have to specify the image. Found None.'''	)

											if not all(task in ['''semantic''', '''instance''', '''panoptic'''] for task in task_inputs	):
																	raise ValueError('''task_inputs must be semantic, instance, or panoptic.'''	)

											lowercase_			:  Optional[int]									= self.image_processor(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)

											if isinstance(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):
																	lowercase_			:  List[str]									= [task_inputs]

											if isinstance(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	) and all(isinstance(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	) for task_input in task_inputs	):
																	lowercase_			:  List[str]									= []
																	for task in task_inputs:
																							lowercase_			:  Tuple									= F'''the task is {task}'''
																							task_token_inputs.append(__SCREAMING_SNAKE_CASE	)
																	lowercase_			:  Tuple									= self._preprocess_text(__SCREAMING_SNAKE_CASE       ,							max_length=self.task_seq_length	)
											else:
																	raise TypeError('''Task Inputs should be a string or a list of strings.'''	)

											if hasattr(__SCREAMING_SNAKE_CASE       ,							'''text_inputs'''	):
																	lowercase_			:  Any									= encoded_inputs.text_inputs

																	lowercase_			:  int									= []
																	for texts in texts_list:
																							lowercase_			:  Any									= self._preprocess_text(__SCREAMING_SNAKE_CASE       ,							max_length=self.max_seq_length	)
																							text_inputs.append(text_input_list.unsqueeze(0	)	)

																	lowercase_			:  Tuple									= torch.cat(__SCREAMING_SNAKE_CASE       ,							dim=0	)

											return encoded_inputs






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE=None       ,							__SCREAMING_SNAKE_CASE=None       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											if task_inputs is None:
																	raise ValueError('''You have to specify the task_input. Found None.'''	)
											elif images is None:
																	raise ValueError('''You have to specify the image. Found None.'''	)

											if not all(task in ['''semantic''', '''instance''', '''panoptic'''] for task in task_inputs	):
																	raise ValueError('''task_inputs must be semantic, instance, or panoptic.'''	)

											lowercase_			:  Dict									= self.image_processor.encode_inputs(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)

											if isinstance(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):
																	lowercase_			:  List[Any]									= [task_inputs]

											if isinstance(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	) and all(isinstance(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	) for task_input in task_inputs	):
																	lowercase_			:  int									= []
																	for task in task_inputs:
																							lowercase_			:  int									= F'''the task is {task}'''
																							task_token_inputs.append(__SCREAMING_SNAKE_CASE	)
																	lowercase_			:  str									= self._preprocess_text(__SCREAMING_SNAKE_CASE       ,							max_length=self.task_seq_length	)
											else:
																	raise TypeError('''Task Inputs should be a string or a list of strings.'''	)

											if hasattr(__SCREAMING_SNAKE_CASE       ,							'''text_inputs'''	):
																	lowercase_			:  Dict									= encoded_inputs.text_inputs

																	lowercase_			:  str									= []
																	for texts in texts_list:
																							lowercase_			:  List[str]									= self._preprocess_text(__SCREAMING_SNAKE_CASE       ,							max_length=self.max_seq_length	)
																							text_inputs.append(text_input_list.unsqueeze(0	)	)

																	lowercase_			:  Union[str, Any]									= torch.cat(__SCREAMING_SNAKE_CASE       ,							dim=0	)

											return encoded_inputs






					def 			_snake_case					(     self       ,							*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											return self.image_processor.post_process_semantic_segmentation(*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self       ,							*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											return self.image_processor.post_process_instance_segmentation(*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self       ,							*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											return self.image_processor.post_process_panoptic_segmentation(*__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)



