




'''simple docstring'''




import unittest

from transformers import ErnieMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    ErnieMForInformationExtraction,
				    ErnieMForMultipleChoice,
				    ErnieMForQuestionAnswering,
				    ErnieMForSequenceClassification,
				    ErnieMForTokenClassification,
				    ErnieMModel,
				)
				from transformers.models.ernie_m.modeling_ernie_m import ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST

class 						lowerCAmelCase_   :







		'''simple docstring'''







		def __init__(						self    :	Optional[int]						,		_UpperCAmelCase    :	Any						,		_UpperCAmelCase    :	Any=13						,		_UpperCAmelCase    :	Optional[int]=7						,		_UpperCAmelCase    :	Dict=True						,		_UpperCAmelCase    :	str=True						,		_UpperCAmelCase    :	Tuple=True						,		_UpperCAmelCase    :	List[str]=99						,		_UpperCAmelCase    :	int=32						,		_UpperCAmelCase    :	int=5						,		_UpperCAmelCase    :	Dict=4						,		_UpperCAmelCase    :	Dict=37						,		_UpperCAmelCase    :	List[Any]="gelu"						,		_UpperCAmelCase    :	Dict=0.1						,		_UpperCAmelCase    :	List[Any]=0.1						,		_UpperCAmelCase    :	List[Any]=5_12						,		_UpperCAmelCase    :	Optional[Any]=16						,		_UpperCAmelCase    :	Tuple=2						,		_UpperCAmelCase    :	List[str]=0.02						,		_UpperCAmelCase    :	List[str]=3						,		_UpperCAmelCase    :	Any=4						,		_UpperCAmelCase    :	Optional[int]=None						,		):






			"""simple docstring"""






			UpperCAmelCase__										=    parent
			UpperCAmelCase__										=    batch_size
			UpperCAmelCase__										=    seq_length
			UpperCAmelCase__										=    is_training
			UpperCAmelCase__										=    use_input_mask
			UpperCAmelCase__										=    use_labels
			UpperCAmelCase__										=    vocab_size
			UpperCAmelCase__										=    hidden_size
			UpperCAmelCase__										=    num_hidden_layers
			UpperCAmelCase__										=    num_attention_heads
			UpperCAmelCase__										=    intermediate_size
			UpperCAmelCase__										=    hidden_act
			UpperCAmelCase__										=    hidden_dropout_prob
			UpperCAmelCase__										=    attention_probs_dropout_prob
			UpperCAmelCase__										=    max_position_embeddings
			UpperCAmelCase__										=    type_vocab_size
			UpperCAmelCase__										=    type_sequence_label_size
			UpperCAmelCase__										=    initializer_range
			UpperCAmelCase__										=    num_labels
			UpperCAmelCase__										=    num_choices
			UpperCAmelCase__										=    scope


		def 							SCREAMING_SNAKE_CASE__			(						self    :	Any ):






			"""simple docstring"""






			UpperCAmelCase__										=    ids_tensor([self.batch_size, self.seq_length]						,		self.vocab_size )

			UpperCAmelCase__										=    None
			if self.use_input_mask:
				UpperCAmelCase__										=    random_attention_mask([self.batch_size, self.seq_length] )

			UpperCAmelCase__										=    None
			UpperCAmelCase__										=    None
			UpperCAmelCase__										=    None
			if self.use_labels:
				UpperCAmelCase__										=    ids_tensor([self.batch_size]						,		self.type_sequence_label_size )
				UpperCAmelCase__										=    ids_tensor([self.batch_size, self.seq_length]						,		self.num_labels )
				UpperCAmelCase__										=    ids_tensor([self.batch_size]						,		self.num_choices )

			UpperCAmelCase__										=    self.get_config()

			return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


		def 							SCREAMING_SNAKE_CASE__			(						self    :	List[str] ):






			"""simple docstring"""






			UpperCAmelCase__										=    ids_tensor([self.batch_size, self.seq_length]						,		self.vocab_size )

			UpperCAmelCase__										=    None
			if self.use_input_mask:
				UpperCAmelCase__										=    random_attention_mask([self.batch_size, self.seq_length] )
			UpperCAmelCase__										=    self.get_config()

			return config, input_ids, input_mask


		def 							SCREAMING_SNAKE_CASE__			(						self    :	int ):






			"""simple docstring"""






			return ErnieMConfig(
			    vocab_size=self.vocab_size						,		hidden_size=self.hidden_size						,		num_hidden_layers=self.num_hidden_layers						,		num_attention_heads=self.num_attention_heads						,		intermediate_size=self.intermediate_size						,		hidden_act=self.hidden_act						,		hidden_dropout_prob=self.hidden_dropout_prob						,		attention_probs_dropout_prob=self.attention_probs_dropout_prob						,		max_position_embeddings=self.max_position_embeddings						,		type_vocab_size=self.type_vocab_size						,		initializer_range=self.initializer_range						,		)


		def 							SCREAMING_SNAKE_CASE__			(						self    :	List[str]						,		_UpperCAmelCase    :	Any						,		_UpperCAmelCase    :	Optional[int]						,		_UpperCAmelCase    :	List[Any]						,		_UpperCAmelCase    :	Optional[int]						,		_UpperCAmelCase    :	Tuple						,		_UpperCAmelCase    :	int ):






			"""simple docstring"""






			UpperCAmelCase__										=    ErnieMModel(config=_UpperCAmelCase )
			model.to(_UpperCAmelCase )
			model.eval()
			UpperCAmelCase__										=    model(_UpperCAmelCase						,		return_dict=_UpperCAmelCase )
			self.parent.assertEqual(result.last_hidden_state.shape						,		(self.batch_size, self.seq_length, self.hidden_size) )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	Dict						,		_UpperCAmelCase    :	Optional[int]						,		_UpperCAmelCase    :	Union[str, Any]						,		_UpperCAmelCase    :	List[str]						,		_UpperCAmelCase    :	Optional[Any]						,		_UpperCAmelCase    :	List[str]						,		_UpperCAmelCase    :	str ):






			"""simple docstring"""






			UpperCAmelCase__										=    ErnieMForQuestionAnswering(config=_UpperCAmelCase )
			model.to(_UpperCAmelCase )
			model.eval()
			UpperCAmelCase__										=    model(
			    _UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		start_positions=_UpperCAmelCase						,		end_positions=_UpperCAmelCase						,		)
			self.parent.assertEqual(result.start_logits.shape						,		(self.batch_size, self.seq_length) )
			self.parent.assertEqual(result.end_logits.shape						,		(self.batch_size, self.seq_length) )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[Any]						,		_UpperCAmelCase    :	Any						,		_UpperCAmelCase    :	Tuple						,		_UpperCAmelCase    :	Dict						,		_UpperCAmelCase    :	Optional[Any]						,		_UpperCAmelCase    :	str						,		_UpperCAmelCase    :	Optional[int] ):






			"""simple docstring"""






			UpperCAmelCase__										=    ErnieMForInformationExtraction(config=_UpperCAmelCase )
			model.to(_UpperCAmelCase )
			model.eval()
			UpperCAmelCase__										=    torch.ones_like(_UpperCAmelCase						,		dtype=torch.floataa )
			UpperCAmelCase__										=    model(
			    _UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		start_positions=_UpperCAmelCase						,		end_positions=_UpperCAmelCase						,		)
			self.parent.assertEqual(result.start_logits.shape						,		(self.batch_size, self.seq_length) )
			self.parent.assertEqual(result.end_logits.shape						,		(self.batch_size, self.seq_length) )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	List[str]						,		_UpperCAmelCase    :	Optional[int]						,		_UpperCAmelCase    :	Any						,		_UpperCAmelCase    :	Any						,		_UpperCAmelCase    :	Tuple						,		_UpperCAmelCase    :	Optional[Any]						,		_UpperCAmelCase    :	List[str] ):






			"""simple docstring"""






			UpperCAmelCase__										=    self.num_labels
			UpperCAmelCase__										=    ErnieMForSequenceClassification(_UpperCAmelCase )
			model.to(_UpperCAmelCase )
			model.eval()
			UpperCAmelCase__										=    model(_UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		labels=_UpperCAmelCase )
			self.parent.assertEqual(result.logits.shape						,		(self.batch_size, self.num_labels) )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	List[str]						,		_UpperCAmelCase    :	str						,		_UpperCAmelCase    :	Any						,		_UpperCAmelCase    :	Optional[int]						,		_UpperCAmelCase    :	int						,		_UpperCAmelCase    :	int						,		_UpperCAmelCase    :	Optional[Any] ):






			"""simple docstring"""






			UpperCAmelCase__										=    self.num_labels
			UpperCAmelCase__										=    ErnieMForTokenClassification(config=_UpperCAmelCase )
			model.to(_UpperCAmelCase )
			model.eval()
			input_ids.to(_UpperCAmelCase )
			input_mask.to(_UpperCAmelCase )
			token_labels.to(_UpperCAmelCase )

			UpperCAmelCase__										=    model(_UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		labels=_UpperCAmelCase )

			self.parent.assertEqual(result.logits.shape						,		(self.batch_size, self.seq_length, self.num_labels) )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	List[Any]						,		_UpperCAmelCase    :	Optional[int]						,		_UpperCAmelCase    :	Dict						,		_UpperCAmelCase    :	List[str]						,		_UpperCAmelCase    :	Optional[Any]						,		_UpperCAmelCase    :	Optional[Any]						,		_UpperCAmelCase    :	List[str] ):






			"""simple docstring"""






			UpperCAmelCase__										=    self.num_choices
			UpperCAmelCase__										=    ErnieMForMultipleChoice(config=_UpperCAmelCase )
			model.to(_UpperCAmelCase )
			model.eval()
			UpperCAmelCase__										=    input_ids.unsqueeze(1 ).expand(-1						,		self.num_choices						,		-1 ).contiguous()
			UpperCAmelCase__										=    input_mask.unsqueeze(1 ).expand(-1						,		self.num_choices						,		-1 ).contiguous()
			UpperCAmelCase__										=    model(
			    _UpperCAmelCase						,		attention_mask=_UpperCAmelCase						,		labels=_UpperCAmelCase						,		)
			self.parent.assertEqual(result.logits.shape						,		(self.batch_size, self.num_choices) )



		def 							SCREAMING_SNAKE_CASE__			(						self    :	Tuple ):






			"""simple docstring"""






			UpperCAmelCase__										=    self.prepare_config_and_inputs()
			(
			    (
			    UpperCAmelCase__			
			)							,			(
			    UpperCAmelCase__			
			)							,			(
			    UpperCAmelCase__			
			)							,			(
			    UpperCAmelCase__			
			)							,			(
			    UpperCAmelCase__			
			)							,			(
			    UpperCAmelCase__			
			)							,						
			)										=    config_and_inputs
			UpperCAmelCase__										=    {"""input_ids""": input_ids, """attention_mask""": input_mask}
			return config, inputs_dict

@require_torch
class 						lowerCAmelCase_   ( lowerCamelCase_			,  lowerCamelCase_			,  unittest.TestCase   ):







		'''simple docstring'''







		lowerCAmelCase_				:					Dict				     =						(
		    (
		        ErnieMModel,
		        ErnieMForMultipleChoice,
		        ErnieMForQuestionAnswering,
		        ErnieMForSequenceClassification,
		        ErnieMForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		lowerCAmelCase_				:					Tuple				     =						()
		lowerCAmelCase_				:					Tuple				     =						(
		    {
		        """feature-extraction""": ErnieMModel,
		        """question-answering""": ErnieMForQuestionAnswering,
		        """text-classification""": ErnieMForSequenceClassification,
		        """token-classification""": ErnieMForTokenClassification,
		        """zero-shot""": ErnieMForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		lowerCAmelCase_				:					Dict				     =						False


		def 							SCREAMING_SNAKE_CASE__			(						self    :	Union[str, Any]						,		_UpperCAmelCase    :	int						,		_UpperCAmelCase    :	List[str]						,		_UpperCAmelCase    :	str						,		_UpperCAmelCase    :	Union[str, Any]						,		_UpperCAmelCase    :	Tuple ):






			"""simple docstring"""






			if pipeline_test_casse_name == "QAPipelineTests":
				return True

			return False


		def 							SCREAMING_SNAKE_CASE__			(						self    :	Union[str, Any] ):






			"""simple docstring"""






			UpperCAmelCase__										=    ErnieMModelTester(self )
			UpperCAmelCase__										=    ConfigTester(self						,		config_class=_UpperCAmelCase						,		hidden_size=37 )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	Dict ):






			"""simple docstring"""






			self.config_tester.run_common_tests()


		def 							SCREAMING_SNAKE_CASE__			(						self    :	str ):






			"""simple docstring"""






			UpperCAmelCase__										=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*_UpperCAmelCase )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[int] ):






			"""simple docstring"""






			UpperCAmelCase__										=    self.model_tester.prepare_config_and_inputs()
			for type in ["absolute", "relative_key", "relative_key_query"]:
				UpperCAmelCase__										=    type
				self.model_tester.create_and_check_model(*_UpperCAmelCase )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	str ):






			"""simple docstring"""






			UpperCAmelCase__										=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	List[Any] ):






			"""simple docstring"""






			UpperCAmelCase__										=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[Any] ):






			"""simple docstring"""






			UpperCAmelCase__										=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	List[Any] ):






			"""simple docstring"""






			UpperCAmelCase__										=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_information_extraction(*_UpperCAmelCase )


		def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[int] ):






			"""simple docstring"""






			UpperCAmelCase__										=    self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase )



		@slow
		def 							SCREAMING_SNAKE_CASE__			(						self    :	Tuple ):






			"""simple docstring"""






			for model_name in ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
				UpperCAmelCase__										=    ErnieMModel.from_pretrained(_UpperCAmelCase )
				self.assertIsNotNone(_UpperCAmelCase )

@require_torch
class 						lowerCAmelCase_   ( unittest.TestCase   ):







		'''simple docstring'''







		@slow
		def 							SCREAMING_SNAKE_CASE__			(						self    :	Optional[int] ):






			"""simple docstring"""






			UpperCAmelCase__										=    ErnieMModel.from_pretrained("""susnato/ernie-m-base_pytorch""" )
			model.eval()
			UpperCAmelCase__										=    torch.tensor([[0, 1, 2, 3, 4, 5]] )
			UpperCAmelCase__										=    model(_UpperCAmelCase )[0]

			# TODO Replace vocab size
			UpperCAmelCase__										=    7_68

			UpperCAmelCase__										=    torch.Size((1, 6, hidden_size) )
			self.assertEqual(output.shape						,		_UpperCAmelCase )

			UpperCAmelCase__										=    torch.tensor(
			    [[[-0.0012, 0.1245, -0.0214], [-0.0742, 0.0244, -0.0771], [-0.0333, 0.1164, -0.1554]]] )

			self.assertTrue(torch.allclose(output[:, :3, :3]						,		_UpperCAmelCase						,		atol=1E-3 ) )






'''simple docstring'''




import os

from apache_beam.io.filesystems import FileSystems
from apache_beam.pipeline import Pipeline

from .logging import get_logger


UpperCAmelCase_       				=      2 << 2_0  # 2mb
UpperCAmelCase_       				=      get_logger(__name__)

class 						lowerCAmelCase_   ( lowerCamelCase_   ):







  '''simple docstring'''







  def 							SCREAMING_SNAKE_CASE__			(						self    :	Any ):






   """simple docstring"""






   UpperCAmelCase__										=    self._options.get_all_options().get("""runner""" )
   return runner in [None, "DirectRunner", "PortableRunner"]

def    _UpperCamelCase      (				SCREAMING_SNAKE_CASE__			:      Any       ,						SCREAMING_SNAKE_CASE__			:      List[Any]       ,						SCREAMING_SNAKE_CASE__			:      Any=False       ):
 '''simple docstring'''





 UpperCAmelCase__										=    FileSystems
 if fs.exists(SCREAMING_SNAKE_CASE__       ):
  if force_upload:
   logger.info(F'''Remote path already exist: {remote_file_path}. Overwriting it as force_upload=True.'''       )
  else:
   logger.info(F'''Remote path already exist: {remote_file_path}. Skipping it as force_upload=False.'''       )
   return
 with fs.create(SCREAMING_SNAKE_CASE__       ) as remote_file:
  with open(SCREAMING_SNAKE_CASE__       ,						"""rb"""       ) as local_file:
   UpperCAmelCase__										=    local_file.read(SCREAMING_SNAKE_CASE__       )
   while chunk:
    remote_file.write(SCREAMING_SNAKE_CASE__       )
    UpperCAmelCase__										=    local_file.read(SCREAMING_SNAKE_CASE__       )







def    _UpperCamelCase      (				SCREAMING_SNAKE_CASE__			:      Dict       ,						SCREAMING_SNAKE_CASE__			:      Any       ,						SCREAMING_SNAKE_CASE__			:      Dict=False       ):
 '''simple docstring'''





 UpperCAmelCase__										=    FileSystems
 if os.path.exists(SCREAMING_SNAKE_CASE__       ):
  if force_download:
   logger.info(F'''Local path already exist: {remote_file_path}. Overwriting it as force_upload=True.'''       )
  else:
   logger.info(F'''Local path already exist: {remote_file_path}. Skipping it as force_upload=False.'''       )
   return
 with fs.open(SCREAMING_SNAKE_CASE__       ) as remote_file:
  with open(SCREAMING_SNAKE_CASE__       ,						"""wb"""       ) as local_file:
   UpperCAmelCase__										=    remote_file.read(SCREAMING_SNAKE_CASE__       )
   while chunk:
    local_file.write(SCREAMING_SNAKE_CASE__       )
    UpperCAmelCase__										=    remote_file.read(SCREAMING_SNAKE_CASE__       )
