




'''simple docstring'''




import unittest
from unittest.util import safe_repr

from transformers import AutoTokenizer, RwkvConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
							    RwkvForCausalLM,
							    RwkvModel,
							)
							from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_0
else:
							snake_case__          =						False






class     UpperCamelCase_     :







						"""simple docstring"""


						def __init__(     self      :       Optional[Any]							, _lowerCamelCase      :       Optional[int]							, _lowerCamelCase      :       Any=14							, _lowerCamelCase      :       str=7							, _lowerCamelCase      :       List[Any]=True							, _lowerCamelCase      :       Dict=False							, _lowerCamelCase      :       List[Any]=True							, _lowerCamelCase      :       Dict=True							, _lowerCamelCase      :       int=True							, _lowerCamelCase      :       int=99							, _lowerCamelCase      :       List[str]=32							, _lowerCamelCase      :       str=5							, _lowerCamelCase      :       Dict=37							, _lowerCamelCase      :       List[Any]="gelu"							, _lowerCamelCase      :       Any=0.1							, _lowerCamelCase      :       List[Any]=0.1							, _lowerCamelCase      :       Optional[int]=512							, _lowerCamelCase      :       int=16							, _lowerCamelCase      :       Dict=2							, _lowerCamelCase      :       List[str]=3							, _lowerCamelCase      :       Optional[int]=4							, _lowerCamelCase      :       List[Any]=None							, ):





											"""simple docstring"""







											A_ :     List[str] =							parent
											A_ :     int =							batch_size
											A_ :     List[Any] =							seq_length
											A_ :     List[Any] =							is_training
											A_ :     Union[str, Any] =							use_token_type_ids
											A_ :     Any =							use_input_mask
											A_ :     str =							use_labels
											A_ :     List[Any] =							use_mc_token_ids
											A_ :     Optional[Any] =							vocab_size
											A_ :     Any =							hidden_size
											A_ :     List[str] =							num_hidden_layers
											A_ :     Union[str, Any] =							intermediate_size
											A_ :     int =							hidden_act
											A_ :     Union[str, Any] =							hidden_dropout_prob
											A_ :     str =							attention_probs_dropout_prob
											A_ :     List[str] =							max_position_embeddings
											A_ :     Union[str, Any] =							type_vocab_size
											A_ :     Any =							type_sequence_label_size
											A_ :     List[str] =							num_labels
											A_ :     Any =							num_choices
											A_ :     Dict =							scope
											A_ :     List[str] =							vocab_size - 1
											A_ :     int =							vocab_size - 1
											A_ :     List[Any] =							vocab_size - 1







						def       _a    (     self      :       List[str]    ):





											"""simple docstring"""







											return RwkvConfig.from_pretrained('''sgugger/rwkv-4-pile-7b'''    )







						def       _a    (     self      :       Optional[Any]							, _lowerCamelCase      :       Optional[int]=False							, _lowerCamelCase      :       List[str]=False							, _lowerCamelCase      :       str=False    ):





											"""simple docstring"""







											A_ :     Tuple =							ids_tensor([self.batch_size, self.seq_length]							, self.vocab_size    )

											A_ :     Tuple =							None
											if self.use_input_mask:
																A_ :     Dict =							random_attention_mask([self.batch_size, self.seq_length]    )

											A_ :     List[Any] =							None
											if self.use_token_type_ids:
																A_ :     List[Any] =							ids_tensor([self.batch_size, self.seq_length]							, self.type_vocab_size    )

											A_ :     Any =							None
											if self.use_mc_token_ids:
																A_ :     List[Any] =							ids_tensor([self.batch_size, self.num_choices]							, self.seq_length    )

											A_ :     List[str] =							None
											A_ :     Union[str, Any] =							None
											A_ :     Optional[int] =							None
											if self.use_labels:
																A_ :     Any =							ids_tensor([self.batch_size]							, self.type_sequence_label_size    )
																A_ :     Optional[Any] =							ids_tensor([self.batch_size, self.seq_length]							, self.num_labels    )
																A_ :     Union[str, Any] =							ids_tensor([self.batch_size]							, self.num_choices    )

											A_ :     Tuple =							self.get_config(
											    gradient_checkpointing=_lowerCamelCase							, scale_attn_by_inverse_layer_idx=_lowerCamelCase							, reorder_and_upcast_attn=_lowerCamelCase							, )

											return (
											    config,
											    input_ids,
											    input_mask,
											    None,
											    token_type_ids,
											    mc_token_ids,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											)







						def       _a    (     self      :       Any							, _lowerCamelCase      :       int=False							, _lowerCamelCase      :       Dict=False							, _lowerCamelCase      :       Any=False    ):





											"""simple docstring"""







											return RwkvConfig(
											    vocab_size=self.vocab_size							, hidden_size=self.hidden_size							, num_hidden_layers=self.num_hidden_layers							, intermediate_size=self.intermediate_size							, activation_function=self.hidden_act							, resid_pdrop=self.hidden_dropout_prob							, attn_pdrop=self.attention_probs_dropout_prob							, n_positions=self.max_position_embeddings							, type_vocab_size=self.type_vocab_size							, use_cache=_lowerCamelCase							, bos_token_id=self.bos_token_id							, eos_token_id=self.eos_token_id							, pad_token_id=self.pad_token_id							, gradient_checkpointing=_lowerCamelCase							, scale_attn_by_inverse_layer_idx=_lowerCamelCase							, reorder_and_upcast_attn=_lowerCamelCase							, )







						def       _a    (     self      :       Union[str, Any]    ):





											"""simple docstring"""







											A_ :     Any =							self.get_config()
											A_ :     str =							300
											return config







						def       _a    (     self      :       Dict    ):





											"""simple docstring"""







											(
											    (
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,
											) :     Union[str, Any] =							self.prepare_config_and_inputs()

											A_ :     Union[str, Any] =							floats_tensor([self.batch_size, self.seq_length, self.hidden_size]    )
											A_ :     List[str] =							ids_tensor([self.batch_size, self.seq_length]							, vocab_size=2    )

											return (
											    config,
											    input_ids,
											    input_mask,
											    head_mask,
											    token_type_ids,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											    encoder_hidden_states,
											    encoder_attention_mask,
											)







						def       _a    (     self      :       List[Any]							, _lowerCamelCase      :       Dict							, _lowerCamelCase      :       Tuple							, _lowerCamelCase      :       Optional[int]							, _lowerCamelCase      :       Optional[Any]							, _lowerCamelCase      :       int							, *_lowerCamelCase      :       List[Any]    ):





											"""simple docstring"""







											A_ :     List[Any] =							True
											A_ :     List[str] =							RwkvModel(config=_lowerCamelCase    )
											model.to(_lowerCamelCase    )
											model.eval()

											A_ :     Any =							model(_lowerCamelCase    )

											self.parent.assertEqual(result.last_hidden_state.shape							, (self.batch_size, self.seq_length, self.hidden_size)    )
											self.parent.assertEqual(len(result.hidden_states    )							, config.num_hidden_layers + 1    )







						def       _a    (     self      :       List[Any]							, _lowerCamelCase      :       str							, _lowerCamelCase      :       Union[str, Any]							, _lowerCamelCase      :       Optional[Any]							, _lowerCamelCase      :       Optional[int]							, _lowerCamelCase      :       List[Any]							, *_lowerCamelCase      :       int    ):





											"""simple docstring"""







											A_ :     List[Any] =							RwkvForCausalLM(_lowerCamelCase    )
											model.to(_lowerCamelCase    )
											model.eval()

											A_ :     Any =							model(_lowerCamelCase							, labels=_lowerCamelCase    )
											self.parent.assertEqual(result.loss.shape							, ()    )
											self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.seq_length, self.vocab_size)    )







						def       _a    (     self      :       Optional[Any]							, _lowerCamelCase      :       Any							, _lowerCamelCase      :       int							, _lowerCamelCase      :       Any							, _lowerCamelCase      :       List[Any]							, _lowerCamelCase      :       Dict							, *_lowerCamelCase      :       Optional[int]    ):





											"""simple docstring"""







											A_ :     Any =							RwkvModel(config=_lowerCamelCase    )
											model.to(_lowerCamelCase    )
											model.eval()

											A_ :     int =							model(_lowerCamelCase    )
											A_ :     str =							outputs.last_hidden_state

											A_ :     str =							model(input_ids[:, :2]    )
											A_ :     Optional[Any] =							outputs.last_hidden_state

											# Using the state computed on the first inputs, we will get the same output
											A_ :     Tuple =							model(input_ids[:, 2:]							, state=outputs.state    )
											A_ :     Optional[int] =							outputs.last_hidden_state

											self.parent.assertTrue(torch.allclose(torch.cat([output_one, output_two]							, dim=1    )							, _lowerCamelCase							, atol=1E-5    )    )







						def       _a    (     self      :       List[Any]							, _lowerCamelCase      :       int							, _lowerCamelCase      :       List[Any]							, _lowerCamelCase      :       Any							, _lowerCamelCase      :       Optional[int]							, _lowerCamelCase      :       List[str]							, *_lowerCamelCase      :       str							, _lowerCamelCase      :       Tuple=False    ):





											"""simple docstring"""







											A_ :     str =							RwkvForCausalLM(_lowerCamelCase    )
											model.to(_lowerCamelCase    )
											if gradient_checkpointing:
																model.gradient_checkpointing_enable()

											A_ :     Any =							model(_lowerCamelCase							, labels=_lowerCamelCase    )
											self.parent.assertEqual(result.loss.shape							, ()    )
											self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.seq_length, self.vocab_size)    )
											result.loss.backward()







						def       _a    (     self      :       str    ):





											"""simple docstring"""







											A_ :     List[Any] =							self.prepare_config_and_inputs()

											(
											    (
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,(
											    A_
											) ,
											) :     Optional[Any] =							config_and_inputs

											A_ :     Optional[Any] =							{'''input_ids''': input_ids}

											return config, inputs_dict






@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0,      reason='See https://github.com/huggingface/transformers/pull/24204'     )
@require_torch
class     UpperCamelCase_     (a__,      a__,      a__,      unittest.TestCase     ):







						"""simple docstring"""


						_lowerCAmelCase    						=					(RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
						_lowerCAmelCase    						=					(
						    {'feature-extraction': RwkvModel, 'text-generation': RwkvForCausalLM} if is_torch_available() else {}
						)
						# all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
						_lowerCAmelCase    						=					False
						_lowerCAmelCase    						=					False
						_lowerCAmelCase    						=					False
						_lowerCAmelCase    						=					False
						_lowerCAmelCase    						=					False  # Rwkv does not support head masking







						def       _a    (     self      :       Optional[int]    ):





											"""simple docstring"""







											A_ :     int =							RwkvModelTester(self    )
											A_ :     Tuple =							ConfigTester(
											    self							, config_class=_lowerCamelCase							, n_embd=37							, common_properties=['''hidden_size''', '''num_hidden_layers''']    )







						def       _a    (     self      :       List[Any]							, _lowerCamelCase      :       Optional[Any]							, _lowerCamelCase      :       Optional[int]							, _lowerCamelCase      :       List[Any]=None    ):





											"""simple docstring"""







											if isinstance(_lowerCamelCase							, torch.Tensor    ):
																A_ ,A_ :     str =							member.max().item(), member.min().item()
											elif isinstance(_lowerCamelCase							, _lowerCamelCase    ) or isinstance(_lowerCamelCase							, _lowerCamelCase    ):
																A_ ,A_ :     List[str] =							max(_lowerCamelCase    ), min(_lowerCamelCase    )

											if not isinstance(_lowerCamelCase							, _lowerCamelCase    ):
																raise TypeError('''container should be a list or tuple'''    )
											elif len(_lowerCamelCase    ) != 2:
																raise ValueError('''container should have 2 elements'''    )

											A_ ,A_ :     List[Any] =							container

											A_ :     Optional[Any] =							(min_value >= expected_min) and (max_value <= expected_max)

											if not is_inside_interval:
																A_ :     int =							'''%s not found in %s''' % (safe_repr(_lowerCamelCase    ), safe_repr(_lowerCamelCase    ))
																self.fail(self._formatMessage(_lowerCamelCase							, _lowerCamelCase    )    )







						def       _a    (     self      :       str    ):





											"""simple docstring"""







											self.config_tester.run_common_tests()







						def       _a    (     self      :       Any    ):





											"""simple docstring"""







											A_ :     Optional[int] =							self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_rwkv_model(*_lowerCamelCase    )







						def       _a    (     self      :       Any    ):





											"""simple docstring"""







											A_ :     Dict =							self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_causl_lm(*_lowerCamelCase    )







						def       _a    (     self      :       Tuple    ):





											"""simple docstring"""







											A_ :     List[Any] =							self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_state_equivalency(*_lowerCamelCase    )







						def       _a    (     self      :       str    ):





											"""simple docstring"""







											A_ ,A_ :     Tuple =							self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																A_ :     str =							model_class(config=_lowerCamelCase    )
																for name, param in model.named_parameters():
																					if "time_decay" in name:
																										if param.requires_grad:
																															self.assertTrue(param.data.max().item() == 3.0    )
																															self.assertTrue(param.data.min().item() == -5.0    )
																					elif "time_first" in name:
																										if param.requires_grad:
																															# check if it's a ones like
																															self.assertTrue(torch.allclose(param.data							, torch.ones_like(param.data    )							, atol=1E-5							, rtol=1E-5    )    )
																					elif any(x in name for x in ['''time_mix_key''', '''time_mix_receptance''']    ):
																										if param.requires_grad:
																															self.assertInterval(
																															    param.data							, [0.0, 1.0]							, msg=f'Parameter {name} of model {model_class} seems not properly initialized'							, )
																					elif "time_mix_value" in name:
																										if param.requires_grad:
																															self.assertInterval(
																															    param.data							, [0.0, 1.3]							, msg=f'Parameter {name} of model {model_class} seems not properly initialized'							, )







						def       _a    (     self      :       Tuple    ):





											"""simple docstring"""







											A_ ,A_ :     Optional[int] =							self.model_tester.prepare_config_and_inputs_for_common()
											A_ :     Optional[int] =							True

											A_ :     List[str] =							getattr(self.model_tester							, '''seq_length'''							, _lowerCamelCase    )

											for model_class in self.all_model_classes:
																A_ :     Optional[Any] =							True
																A_ :     List[Any] =							False
																A_ :     Dict =							True
																A_ :     Any =							model_class(_lowerCamelCase    )
																model.to(_lowerCamelCase    )
																model.eval()

																A_ :     int =							self._prepare_for_class(_lowerCamelCase							, _lowerCamelCase    )
																A_ :     Tuple =							inputs['''input_ids'''].shape[0]
																with torch.no_grad():
																					A_ :     Union[str, Any] =							model(**_lowerCamelCase    )
																A_ :     str =							outputs.attentions
																self.assertEqual(len(_lowerCamelCase    )							, self.model_tester.num_hidden_layers    )

																# check that output_attentions also work using config
																del inputs_dict["output_attentions"]
																A_ :     Optional[int] =							True
																A_ :     Any =							model_class(_lowerCamelCase    )
																model.to(_lowerCamelCase    )
																model.eval()

																A_ :     Dict =							self._prepare_for_class(_lowerCamelCase							, _lowerCamelCase    )
																A_ :     Optional[Any] =							inputs['''input_ids'''].shape[0]
																with torch.no_grad():
																					A_ :     str =							model(**_lowerCamelCase    )
																A_ :     Tuple =							outputs.attentions
																self.assertEqual(len(_lowerCamelCase    )							, self.model_tester.num_hidden_layers    )

																self.assertListEqual(
																    list(attentions[0].shape[-3:]    )							, [batch_size, seq_len, config.hidden_size]							, )
																A_ :     Any =							len(_lowerCamelCase    )

																# Check attention is always last and order is fine
																A_ :     Optional[int] =							True
																A_ :     int =							True
																A_ :     Optional[int] =							model_class(_lowerCamelCase    )
																model.to(_lowerCamelCase    )
																model.eval()

																A_ :     int =							self._prepare_for_class(_lowerCamelCase							, _lowerCamelCase    )
																A_ :     str =							inputs['''input_ids'''].shape[0]
																with torch.no_grad():
																					A_ :     Any =							model(**_lowerCamelCase    )

																A_ :     List[str] =							1
																self.assertEqual(out_len + added_hidden_states							, len(_lowerCamelCase    )    )

																A_ :     int =							outputs.attentions

																self.assertEqual(len(_lowerCamelCase    )							, self.model_tester.num_hidden_layers    )
																self.assertListEqual(
																    list(self_attentions[0].shape[-3:]    )							, [batch_size, seq_len, config.hidden_size]							, )







						@slow
						def       _a    (     self      :       int    ):





											"""simple docstring"""







											for model_name in RWKV_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																A_ :     Any =							RwkvModel.from_pretrained(_lowerCamelCase    )
																self.assertIsNotNone(_lowerCamelCase    )






@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0,      reason='See https://github.com/huggingface/transformers/pull/24204'     )
@slow
class     UpperCamelCase_     (unittest.TestCase     ):







						"""simple docstring"""


						def       _a    (     self      :       List[str]    ):





											"""simple docstring"""







											A_ :     Optional[Any] =							'''RWKV/rwkv-4-169m-pile'''
											A_ :     List[str] =							AutoTokenizer.from_pretrained(self.model_id    )







						def       _a    (     self      :       Union[str, Any]    ):





											"""simple docstring"""







											A_ :     List[str] =							'''Hello my name is Jasmine and I am a newbie to the'''
											A_ :     Dict =							RwkvForCausalLM.from_pretrained(self.model_id    ).to(_lowerCamelCase    )

											A_ :     str =							self.tokenizer('''Hello my name is'''							, return_tensors='''pt'''    ).input_ids.to(_lowerCamelCase    )
											A_ :     List[Any] =							model.generate(_lowerCamelCase							, max_new_tokens=10    )
											A_ :     Tuple =							self.tokenizer.decode(output[0].tolist()    )

											self.assertEqual(_lowerCamelCase							, _lowerCamelCase    )







						def       _a    (     self      :       Optional[int]    ):





											"""simple docstring"""







											A_ :     Dict =							'''Hello my name is Jasmine and I am a newbie to the'''

											A_ :     Optional[int] =							self.tokenizer('''Hello my name is'''							, return_tensors='''pt'''    ).input_ids.to(_lowerCamelCase    )
											A_ :     Optional[int] =							RwkvForCausalLM.from_pretrained(self.model_id							, torch_dtype=torch.bfloataa    ).to(_lowerCamelCase    )

											A_ :     Optional[Any] =							model.generate(_lowerCamelCase							, max_new_tokens=10    )
											A_ :     Optional[int] =							self.tokenizer.decode(output[0].tolist()    )

											self.assertEqual(_lowerCamelCase							, _lowerCamelCase    )









'''simple docstring'''




def 						snake_case__    ( lowerCamelCase__				:   float			)   ->   float:
					return -num if num < 0 else num




def 						snake_case__    ( lowerCamelCase__				:   list[int]			)   ->   int:
					if len(lowerCamelCase__			) == 0:
										raise ValueError('''abs_min() arg is an empty sequence'''			)
					A_ :     Any =							x[0]
					for i in x:
										if abs_val(lowerCamelCase__			) < abs_val(lowerCamelCase__			):
															A_ :     Tuple =							i
					return j




def 						snake_case__    ( lowerCamelCase__				:   list[int]			)   ->   int:
					if len(lowerCamelCase__			) == 0:
										raise ValueError('''abs_max() arg is an empty sequence'''			)
					A_ :     List[str] =							x[0]
					for i in x:
										if abs(lowerCamelCase__			) > abs(lowerCamelCase__			):
															A_ :     Any =							i
					return j




def 						snake_case__    ( lowerCamelCase__				:   list[int]			)   ->   int:
					if len(lowerCamelCase__			) == 0:
										raise ValueError('''abs_max_sort() arg is an empty sequence'''			)
					return sorted(lowerCamelCase__			,		key=lowerCamelCase__			)[-1]




def 						snake_case__    ( )   ->   Optional[Any]:
					assert abs_val(0			) == 0
					assert abs_val(3_4			) == 3_4
					assert abs_val(-1_0_0_0_0_0_0_0_0_0_0_0			) == 1_0_0_0_0_0_0_0_0_0_0_0

					A_ :     Dict =							[-3, -1, 2, -1_1]
					assert abs_max(lowerCamelCase__			) == -1_1
					assert abs_max_sort(lowerCamelCase__			) == -1_1
					assert abs_min(lowerCamelCase__			) == -1


if __name__ == "__main__":
							import doctest

							doctest.testmod()

							test_abs_val()
							print(abs_val(-34))  # --> 34



