
'''simple docstring'''







import unittest

from transformers import ErnieMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    ErnieMForInformationExtraction,
							    ErnieMForMultipleChoice,
							    ErnieMForQuestionAnswering,
							    ErnieMForSequenceClassification,
							    ErnieMForTokenClassification,
							    ErnieMModel,
							)
							from transformers.models.ernie_m.modeling_ernie_m import ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST






class 				lowerCAmelCase__     :






					def __init__(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE=13       ,							__SCREAMING_SNAKE_CASE=7       ,							__SCREAMING_SNAKE_CASE=True       ,							__SCREAMING_SNAKE_CASE=True       ,							__SCREAMING_SNAKE_CASE=True       ,							__SCREAMING_SNAKE_CASE=99       ,							__SCREAMING_SNAKE_CASE=32       ,							__SCREAMING_SNAKE_CASE=5       ,							__SCREAMING_SNAKE_CASE=4       ,							__SCREAMING_SNAKE_CASE=37       ,							__SCREAMING_SNAKE_CASE="gelu"       ,							__SCREAMING_SNAKE_CASE=0.1       ,							__SCREAMING_SNAKE_CASE=0.1       ,							__SCREAMING_SNAKE_CASE=5_12       ,							__SCREAMING_SNAKE_CASE=16       ,							__SCREAMING_SNAKE_CASE=2       ,							__SCREAMING_SNAKE_CASE=0.02       ,							__SCREAMING_SNAKE_CASE=3       ,							__SCREAMING_SNAKE_CASE=4       ,							__SCREAMING_SNAKE_CASE=None       ,							):


											"""simple docstring"""





											lowercase_			:  Optional[int]									= parent
											lowercase_			:  Tuple									= batch_size
											lowercase_			:  Tuple									= seq_length
											lowercase_			:  str									= is_training
											lowercase_			:  Optional[Any]									= use_input_mask
											lowercase_			:  Optional[Any]									= use_labels
											lowercase_			:  int									= vocab_size
											lowercase_			:  Tuple									= hidden_size
											lowercase_			:  Tuple									= num_hidden_layers
											lowercase_			:  Dict									= num_attention_heads
											lowercase_			:  Union[str, Any]									= intermediate_size
											lowercase_			:  List[str]									= hidden_act
											lowercase_			:  Any									= hidden_dropout_prob
											lowercase_			:  Union[str, Any]									= attention_probs_dropout_prob
											lowercase_			:  str									= max_position_embeddings
											lowercase_			:  Optional[Any]									= type_vocab_size
											lowercase_			:  List[str]									= type_sequence_label_size
											lowercase_			:  int									= initializer_range
											lowercase_			:  List[Any]									= num_labels
											lowercase_			:  Union[str, Any]									= num_choices
											lowercase_			:  Union[str, Any]									= scope






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Optional[Any]									= ids_tensor([self.batch_size, self.seq_length]       ,							self.vocab_size	)

											lowercase_			:  List[Any]									= None
											if self.use_input_mask:
																	lowercase_			:  Optional[Any]									= random_attention_mask([self.batch_size, self.seq_length]	)

											lowercase_			:  Optional[Any]									= None
											lowercase_			:  Dict									= None
											lowercase_			:  Any									= None
											if self.use_labels:
																	lowercase_			:  Optional[Any]									= ids_tensor([self.batch_size]       ,							self.type_sequence_label_size	)
																	lowercase_			:  Any									= ids_tensor([self.batch_size, self.seq_length]       ,							self.num_labels	)
																	lowercase_			:  Dict									= ids_tensor([self.batch_size]       ,							self.num_choices	)

											lowercase_			:  int									= self.get_config()

											return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Dict									= ids_tensor([self.batch_size, self.seq_length]       ,							self.vocab_size	)

											lowercase_			:  int									= None
											if self.use_input_mask:
																	lowercase_			:  Optional[Any]									= random_attention_mask([self.batch_size, self.seq_length]	)
											lowercase_			:  List[Any]									= self.get_config()

											return config, input_ids, input_mask






					def 			_snake_case					(     self	):


											"""simple docstring"""





											return ErnieMConfig(
											    vocab_size=self.vocab_size       ,							hidden_size=self.hidden_size       ,							num_hidden_layers=self.num_hidden_layers       ,							num_attention_heads=self.num_attention_heads       ,							intermediate_size=self.intermediate_size       ,							hidden_act=self.hidden_act       ,							hidden_dropout_prob=self.hidden_dropout_prob       ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,							max_position_embeddings=self.max_position_embeddings       ,							type_vocab_size=self.type_vocab_size       ,							initializer_range=self.initializer_range       ,							)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  List[str]									= ErnieMModel(config=__SCREAMING_SNAKE_CASE	)
											model.to(__SCREAMING_SNAKE_CASE	)
											model.eval()
											lowercase_			:  Tuple									= model(__SCREAMING_SNAKE_CASE       ,							return_dict=__SCREAMING_SNAKE_CASE	)
											self.parent.assertEqual(result.last_hidden_state.shape       ,							(self.batch_size, self.seq_length, self.hidden_size)	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  List[Any]									= ErnieMForQuestionAnswering(config=__SCREAMING_SNAKE_CASE	)
											model.to(__SCREAMING_SNAKE_CASE	)
											model.eval()
											lowercase_			:  Optional[Any]									= model(
											    __SCREAMING_SNAKE_CASE       ,							attention_mask=__SCREAMING_SNAKE_CASE       ,							start_positions=__SCREAMING_SNAKE_CASE       ,							end_positions=__SCREAMING_SNAKE_CASE       ,							)
											self.parent.assertEqual(result.start_logits.shape       ,							(self.batch_size, self.seq_length)	)
											self.parent.assertEqual(result.end_logits.shape       ,							(self.batch_size, self.seq_length)	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  str									= ErnieMForInformationExtraction(config=__SCREAMING_SNAKE_CASE	)
											model.to(__SCREAMING_SNAKE_CASE	)
											model.eval()
											lowercase_			:  Any									= torch.ones_like(__SCREAMING_SNAKE_CASE       ,							dtype=torch.floataa	)
											lowercase_			:  Dict									= model(
											    __SCREAMING_SNAKE_CASE       ,							attention_mask=__SCREAMING_SNAKE_CASE       ,							start_positions=__SCREAMING_SNAKE_CASE       ,							end_positions=__SCREAMING_SNAKE_CASE       ,							)
											self.parent.assertEqual(result.start_logits.shape       ,							(self.batch_size, self.seq_length)	)
											self.parent.assertEqual(result.end_logits.shape       ,							(self.batch_size, self.seq_length)	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  str									= self.num_labels
											lowercase_			:  Dict									= ErnieMForSequenceClassification(__SCREAMING_SNAKE_CASE	)
											model.to(__SCREAMING_SNAKE_CASE	)
											model.eval()
											lowercase_			:  Optional[Any]									= model(__SCREAMING_SNAKE_CASE       ,							attention_mask=__SCREAMING_SNAKE_CASE       ,							labels=__SCREAMING_SNAKE_CASE	)
											self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.num_labels)	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  Optional[Any]									= self.num_labels
											lowercase_			:  Optional[Any]									= ErnieMForTokenClassification(config=__SCREAMING_SNAKE_CASE	)
											model.to(__SCREAMING_SNAKE_CASE	)
											model.eval()
											input_ids.to(__SCREAMING_SNAKE_CASE	)
											input_mask.to(__SCREAMING_SNAKE_CASE	)
											token_labels.to(__SCREAMING_SNAKE_CASE	)

											lowercase_			:  Optional[Any]									= model(__SCREAMING_SNAKE_CASE       ,							attention_mask=__SCREAMING_SNAKE_CASE       ,							labels=__SCREAMING_SNAKE_CASE	)

											self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.seq_length, self.num_labels)	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  int									= self.num_choices
											lowercase_			:  int									= ErnieMForMultipleChoice(config=__SCREAMING_SNAKE_CASE	)
											model.to(__SCREAMING_SNAKE_CASE	)
											model.eval()
											lowercase_			:  Optional[Any]									= input_ids.unsqueeze(1	).expand(-1       ,							self.num_choices       ,							-1	).contiguous()
											lowercase_			:  Optional[int]									= input_mask.unsqueeze(1	).expand(-1       ,							self.num_choices       ,							-1	).contiguous()
											lowercase_			:  Optional[Any]									= model(
											    __SCREAMING_SNAKE_CASE       ,							attention_mask=__SCREAMING_SNAKE_CASE       ,							labels=__SCREAMING_SNAKE_CASE       ,							)
											self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.num_choices)	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Any									= self.prepare_config_and_inputs()
											(
											    (
											    lowercase_
											)		,				(
											    lowercase_
											)		,				(
											    lowercase_
											)		,				(
											    lowercase_
											)		,				(
											    lowercase_
											)		,				(
											    lowercase_
											)		,				
											)			:  Union[str, Any]									= config_and_inputs
											lowercase_			:  Optional[int]									= {'''input_ids''': input_ids, '''attention_mask''': input_mask}
											return config, inputs_dict







@require_torch
class 				lowerCAmelCase__     (						lowerCamelCase_					,		lowerCamelCase_					,		unittest.TestCase  ):
					lowerCAmelCase_       =  (
					    (
					        ErnieMModel,
					        ErnieMForMultipleChoice,
					        ErnieMForQuestionAnswering,
					        ErnieMForSequenceClassification,
					        ErnieMForTokenClassification,
					    )
					    if is_torch_available()
					    else ()
					)
					lowerCAmelCase_       =  ()
					lowerCAmelCase_       =  (
					    {
					        '''feature-extraction''': ErnieMModel,
					        '''question-answering''': ErnieMForQuestionAnswering,
					        '''text-classification''': ErnieMForSequenceClassification,
					        '''token-classification''': ErnieMForTokenClassification,
					        '''zero-shot''': ErnieMForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					lowerCAmelCase_       =  False






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											if pipeline_test_casse_name == "QAPipelineTests":
																	return True

											return False






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Optional[Any]									= ErnieMModelTester(self	)
											lowercase_			:  Dict									= ConfigTester(self       ,							config_class=__SCREAMING_SNAKE_CASE       ,							hidden_size=37	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											self.config_tester.run_common_tests()






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Optional[Any]									= self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  List[Any]									= self.model_tester.prepare_config_and_inputs()
											for type in ["absolute", "relative_key", "relative_key_query"]:
																	lowercase_			:  Any									= type
																	self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  str									= self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Any									= self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Tuple									= self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  List[str]									= self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_information_extraction(*__SCREAMING_SNAKE_CASE	)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  Dict									= self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE	)






					@slow
					def 			_snake_case					(     self	):


											"""simple docstring"""





											for model_name in ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	lowercase_			:  List[Any]									= ErnieMModel.from_pretrained(__SCREAMING_SNAKE_CASE	)
																	self.assertIsNotNone(__SCREAMING_SNAKE_CASE	)







@require_torch
class 				lowerCAmelCase__     (						unittest.TestCase  ):






					@slow
					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  List[str]									= ErnieMModel.from_pretrained('''susnato/ernie-m-base_pytorch'''	)
											model.eval()
											lowercase_			:  str									= torch.tensor([[0, 1, 2, 3, 4, 5]]	)
											lowercase_			:  Dict									= model(__SCREAMING_SNAKE_CASE	)[0]

											# TODO Replace vocab size
											lowercase_			:  Any									= 7_68

											lowercase_			:  Tuple									= torch.Size((1, 6, hidden_size)	)
											self.assertEqual(output.shape       ,							__SCREAMING_SNAKE_CASE	)

											lowercase_			:  str									= torch.tensor(
											    [[[-0.0_012, 0.1_245, -0.0_214], [-0.0_742, 0.0_244, -0.0_771], [-0.0_333, 0.1_164, -0.1_554]]]	)

											self.assertTrue(torch.allclose(output[:, :3, :3]       ,							__SCREAMING_SNAKE_CASE       ,							atol=1E-3	)	)





'''simple docstring'''







import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






class 				lowerCAmelCase__     (						lowerCamelCase_  ):






					def __init__(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = False       ,							__SCREAMING_SNAKE_CASE = False       ,							__SCREAMING_SNAKE_CASE = None       ,							**__SCREAMING_SNAKE_CASE       ,							):


											"""simple docstring"""





											super().__init__(
											    __SCREAMING_SNAKE_CASE       ,							split=__SCREAMING_SNAKE_CASE       ,							features=__SCREAMING_SNAKE_CASE       ,							cache_dir=__SCREAMING_SNAKE_CASE       ,							keep_in_memory=__SCREAMING_SNAKE_CASE       ,							streaming=__SCREAMING_SNAKE_CASE       ,							num_proc=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE       ,							)
											lowercase_			:  int									= path_or_paths if isinstance(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	) else {self.split: path_or_paths}
											lowercase_			:  List[Any]									= Csv(
											    cache_dir=__SCREAMING_SNAKE_CASE       ,							data_files=__SCREAMING_SNAKE_CASE       ,							features=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE       ,							)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											if self.streaming:
																	lowercase_			:  List[str]									= self.builder.as_streaming_dataset(split=self.split	)
											# Build regular (map-style) dataset
											else:
																	lowercase_			:  Any									= None
																	lowercase_			:  Optional[Any]									= None
																	lowercase_			:  str									= None
																	lowercase_			:  Optional[int]									= None

																	self.builder.download_and_prepare(
																	    download_config=__SCREAMING_SNAKE_CASE       ,							download_mode=__SCREAMING_SNAKE_CASE       ,							verification_mode=__SCREAMING_SNAKE_CASE       ,							base_path=__SCREAMING_SNAKE_CASE       ,							num_proc=self.num_proc       ,							)
																	lowercase_			:  Tuple									= self.builder.as_dataset(
																	    split=self.split       ,							verification_mode=__SCREAMING_SNAKE_CASE       ,							in_memory=self.keep_in_memory	)
											return dataset







class 				lowerCAmelCase__     :






					def __init__(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = None       ,							**__SCREAMING_SNAKE_CASE       ,							):


											"""simple docstring"""





											if num_proc is not None and num_proc <= 0:
																	raise ValueError(F'''num_proc {num_proc} must be an integer > 0.'''	)

											lowercase_			:  List[Any]									= dataset
											lowercase_			:  Dict									= path_or_buf
											lowercase_			:  int									= batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
											lowercase_			:  int									= num_proc
											lowercase_			:  Optional[int]									= '''utf-8'''
											lowercase_			:  Optional[Any]									= to_csv_kwargs






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  List[str]									= self.to_csv_kwargs.pop('''path_or_buf'''       ,							__SCREAMING_SNAKE_CASE	)
											lowercase_			:  Optional[Any]									= self.to_csv_kwargs.pop('''header'''       ,							__SCREAMING_SNAKE_CASE	)
											lowercase_			:  Dict									= self.to_csv_kwargs.pop('''index'''       ,							__SCREAMING_SNAKE_CASE	)

											if isinstance(self.path_or_buf       ,							(str, bytes, os.PathLike)	):
																	with open(self.path_or_buf       ,							'''wb+'''	) as buffer:
																							lowercase_			:  Any									= self._write(file_obj=__SCREAMING_SNAKE_CASE       ,							header=__SCREAMING_SNAKE_CASE       ,							index=__SCREAMING_SNAKE_CASE       ,							**self.to_csv_kwargs	)
											else:
																	lowercase_			:  str									= self._write(file_obj=self.path_or_buf       ,							header=__SCREAMING_SNAKE_CASE       ,							index=__SCREAMING_SNAKE_CASE       ,							**self.to_csv_kwargs	)
											return written






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_		,				lowercase_		,				lowercase_		,				lowercase_			:  int									= args

											lowercase_			:  List[str]									= query_table(
											    table=self.dataset.data       ,							key=slice(__SCREAMING_SNAKE_CASE       ,							offset + self.batch_size	)       ,							indices=self.dataset._indices       ,							)
											lowercase_			:  Dict									= batch.to_pandas().to_csv(
											    path_or_buf=__SCREAMING_SNAKE_CASE       ,							header=header if (offset == 0) else False       ,							index=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)
											return csv_str.encode(self.encoding	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  Dict									= 0

											if self.num_proc is None or self.num_proc == 1:
																	for offset in logging.tqdm(
																	    range(0       ,							len(self.dataset	)       ,							self.batch_size	)       ,							unit='''ba'''       ,							disable=not logging.is_progress_bar_enabled()       ,							desc='''Creating CSV from Arrow format'''       ,							):
																							lowercase_			:  Any									= self._batch_csv((offset, header, index, to_csv_kwargs)	)
																							written += file_obj.write(__SCREAMING_SNAKE_CASE	)

											else:
																	lowercase_		,				lowercase_			:  Union[str, Any]									= len(self.dataset	), self.batch_size
																	with multiprocessing.Pool(self.num_proc	) as pool:
																							for csv_str in logging.tqdm(
																							    pool.imap(
																							        self._batch_csv       ,							[(offset, header, index, to_csv_kwargs) for offset in range(0       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	)]       ,							)       ,							total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size       ,							unit='''ba'''       ,							disable=not logging.is_progress_bar_enabled()       ,							desc='''Creating CSV from Arrow format'''       ,							):
																													written += file_obj.write(__SCREAMING_SNAKE_CASE	)

											return written



