

"""simple docstring"""






import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp
def       UpperCamelCase					(			_lowerCAmelCase     :						str,  _lowerCAmelCase     :						Optional[Any],  _lowerCAmelCase     :						Tuple,  _lowerCAmelCase     :						Tuple,  _lowerCAmelCase     :						int = 4096 )				->			List[str]:
	_UpperCAmelCase						,   _UpperCAmelCase						,   _UpperCAmelCase    :							Tuple									=					key.shape[-3:]
	_UpperCAmelCase    :							List[str]									=					value.shape[-1]
	_UpperCAmelCase    :							str									=					min(_lowerCAmelCase,  _lowerCAmelCase )
	_UpperCAmelCase    :							Tuple									=					query / jnp.sqrt(_lowerCAmelCase )

	@functools.partial(jax.checkpoint,  prevent_cse=_lowerCAmelCase )
	def summarize_chunk(_lowerCAmelCase     :						int,  _lowerCAmelCase     :						Dict,  _lowerCAmelCase     :						Union[str, Any] ):
		_UpperCAmelCase    :							str									=					jnp.einsum("""...qhd,...khd->...qhk""",  _lowerCAmelCase,  _lowerCAmelCase,  precision=_lowerCAmelCase )

		_UpperCAmelCase    :							Any									=					jnp.max(_lowerCAmelCase,  axis=-1,  keepdims=_lowerCAmelCase )
		_UpperCAmelCase    :							str									=					jax.lax.stop_gradient(_lowerCAmelCase )
		_UpperCAmelCase    :							Optional[int]									=					jnp.exp(attn_weights - max_score )

		_UpperCAmelCase    :							Optional[int]									=					jnp.einsum("""...vhf,...qhv->...qhf""",  _lowerCAmelCase,  _lowerCAmelCase,  precision=_lowerCAmelCase )
		_UpperCAmelCase    :							int									=					jnp.einsum("""...qhk->...qh""",  _lowerCAmelCase )

		return (exp_values, exp_weights.sum(axis=-1 ), max_score)

	def chunk_scanner(_lowerCAmelCase     :						List[Any] ):
		# julienne key array
		_UpperCAmelCase    :							Any									=					jax.lax.dynamic_slice(
		    operand=_lowerCAmelCase,  start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0],  slice_sizes=list(key.shape[:-3] ) + [key_chunk_size, num_heads, k_features],  )

		# julienne value array
		_UpperCAmelCase    :							List[str]									=					jax.lax.dynamic_slice(
		    operand=_lowerCAmelCase,  start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0],  slice_sizes=list(value.shape[:-3] ) + [key_chunk_size, num_heads, v_features],  )

		return summarize_chunk(_lowerCAmelCase,  _lowerCAmelCase,  _lowerCAmelCase )

	_UpperCAmelCase						,   _UpperCAmelCase						,   _UpperCAmelCase    :							Any									=					jax.lax.map(f=_lowerCAmelCase,  xs=jnp.arange(0,  _lowerCAmelCase,  _lowerCAmelCase ) )

	_UpperCAmelCase    :							List[Any]									=					jnp.max(_lowerCAmelCase,  axis=0,  keepdims=_lowerCAmelCase )
	_UpperCAmelCase    :							Tuple									=					jnp.exp(chunk_max - global_max )

	chunk_values *= jnp.expand_dims(_lowerCAmelCase,  axis=-1 )
	chunk_weights *= max_diffs

	_UpperCAmelCase    :							Any									=					chunk_values.sum(axis=0 )
	_UpperCAmelCase    :							int									=					jnp.expand_dims(_lowerCAmelCase,  -1 ).sum(axis=0 )

	return all_values / all_weights
def       UpperCamelCase					(			_lowerCAmelCase     :						Tuple,  _lowerCAmelCase     :						Union[str, Any],  _lowerCAmelCase     :						Optional[Any],  _lowerCAmelCase     :						Optional[Any]=jax.lax.Precision.HIGHEST,  _lowerCAmelCase     :						int = 1024,  _lowerCAmelCase     :						int = 4096 )				->			Union[str, Any]:
	_UpperCAmelCase						,   _UpperCAmelCase						,   _UpperCAmelCase    :							Union[str, Any]									=					query.shape[-3:]

	def chunk_scanner(_lowerCAmelCase     :						str,  _lowerCAmelCase     :						str ):
		# julienne query array
		_UpperCAmelCase    :							List[str]									=					jax.lax.dynamic_slice(
		    operand=_lowerCAmelCase,  start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0],  slice_sizes=list(query.shape[:-3] ) + [min(_lowerCAmelCase,  _lowerCAmelCase ), num_heads, q_features],  )

		return (
		    chunk_idx + query_chunk_size,  # unused ignore it
		    _query_chunk_attention(
		        query=_lowerCAmelCase,  key=_lowerCAmelCase,  value=_lowerCAmelCase,  precision=_lowerCAmelCase,  key_chunk_size=_lowerCAmelCase ),
		)

	_UpperCAmelCase						,   _UpperCAmelCase    :							List[Any]									=					jax.lax.scan(
	    f=_lowerCAmelCase,  init=0,  xs=_lowerCAmelCase,  length=math.ceil(num_q / query_chunk_size ) )

	return jnp.concatenate(_lowerCAmelCase,  axis=-3 )  # fuse the chunked result back






class     _UpperCAmelCase			(		nn.Module):
			__a :       int
			__a :       int									=							8
			__a :       int									=							6_4
			__a :       float									=							0.0
			__a :       bool									=							False
			__a :       jnp.dtype									=							jnp.floataa
			def 							__snake_case (		self     )    ->    Optional[int]:






				'''simple docstring'''







				_UpperCAmelCase    :							str									=					self.dim_head * self.heads
				_UpperCAmelCase    :							List[Any]									=					self.dim_head**-0.5

				# Weights were exported with old names {to_q, to_k, to_v, to_out}
				_UpperCAmelCase    :							Union[str, Any]									=					nn.Dense(_A      ,		use_bias=_A      ,		dtype=self.dtype      ,		name="""to_q"""     )
				_UpperCAmelCase    :							str									=					nn.Dense(_A      ,		use_bias=_A      ,		dtype=self.dtype      ,		name="""to_k"""     )
				_UpperCAmelCase    :							Optional[Any]									=					nn.Dense(_A      ,		use_bias=_A      ,		dtype=self.dtype      ,		name="""to_v"""     )

				_UpperCAmelCase    :							List[Any]									=					nn.Dense(self.query_dim      ,		dtype=self.dtype      ,		name="""to_out_0"""     )
				_UpperCAmelCase    :							Any									=					nn.Dropout(rate=self.dropout     )
			def 							__snake_case (		self      ,		_A     )    ->    int:






				'''simple docstring'''







				_UpperCAmelCase						,   _UpperCAmelCase						,   _UpperCAmelCase    :							Any									=					tensor.shape
				_UpperCAmelCase    :							Union[str, Any]									=					self.heads
				_UpperCAmelCase    :							List[str]									=					tensor.reshape(_A      ,		_A      ,		_A      ,		dim // head_size     )
				_UpperCAmelCase    :							Optional[int]									=					jnp.transpose(_A      ,		(0, 2, 1, 3)     )
				_UpperCAmelCase    :							Optional[Any]									=					tensor.reshape(batch_size * head_size      ,		_A      ,		dim // head_size     )
				return tensor
			def 							__snake_case (		self      ,		_A     )    ->    Union[str, Any]:






				'''simple docstring'''







				_UpperCAmelCase						,   _UpperCAmelCase						,   _UpperCAmelCase    :							List[Any]									=					tensor.shape
				_UpperCAmelCase    :							Optional[int]									=					self.heads
				_UpperCAmelCase    :							str									=					tensor.reshape(batch_size // head_size      ,		_A      ,		_A      ,		_A     )
				_UpperCAmelCase    :							Optional[Any]									=					jnp.transpose(_A      ,		(0, 2, 1, 3)     )
				_UpperCAmelCase    :							Tuple									=					tensor.reshape(batch_size // head_size      ,		_A      ,		dim * head_size     )
				return tensor
			def __call__(		self      ,		_A      ,		_A=None      ,		_A=True     )    ->    Dict:






				'''simple docstring'''







				_UpperCAmelCase    :							Optional[int]									=					hidden_states if context is None else context

				_UpperCAmelCase    :							List[Any]									=					self.query(_A     )
				_UpperCAmelCase    :							List[Any]									=					self.key(_A     )
				_UpperCAmelCase    :							Union[str, Any]									=					self.value(_A     )

				_UpperCAmelCase    :							List[str]									=					self.reshape_heads_to_batch_dim(_A     )
				_UpperCAmelCase    :							List[Any]									=					self.reshape_heads_to_batch_dim(_A     )
				_UpperCAmelCase    :							List[Any]									=					self.reshape_heads_to_batch_dim(_A     )

				if self.use_memory_efficient_attention:
					_UpperCAmelCase    :							List[Any]									=					query_states.transpose(1      ,		0      ,		2     )
					_UpperCAmelCase    :							Any									=					key_states.transpose(1      ,		0      ,		2     )
					_UpperCAmelCase    :							Optional[int]									=					value_states.transpose(1      ,		0      ,		2     )

					# this if statement create a chunk size for each layer of the unet
					# the chunk size is equal to the query_length dimension of the deepest layer of the unet

					_UpperCAmelCase    :							int									=					query_states.shape[-3]
					if flatten_latent_dim % 64 == 0:
						_UpperCAmelCase    :							Union[str, Any]									=					int(flatten_latent_dim / 64     )
					elif flatten_latent_dim % 16 == 0:
						_UpperCAmelCase    :							List[str]									=					int(flatten_latent_dim / 16     )
					elif flatten_latent_dim % 4 == 0:
						_UpperCAmelCase    :							Optional[int]									=					int(flatten_latent_dim / 4     )
					else:
						_UpperCAmelCase    :							Union[str, Any]									=					int(_A     )

					_UpperCAmelCase    :							Dict									=					jax_memory_efficient_attention(
					    _A      ,		_A      ,		_A      ,		query_chunk_size=_A      ,		key_chunk_size=40_96 * 4     )

					_UpperCAmelCase    :							Dict									=					hidden_states.transpose(1      ,		0      ,		2     )
				else:
					# compute attentions
					_UpperCAmelCase    :							List[Any]									=					jnp.einsum("""b i d, b j d->b i j"""      ,		_A      ,		_A     )
					_UpperCAmelCase    :							List[str]									=					attention_scores * self.scale
					_UpperCAmelCase    :							Union[str, Any]									=					nn.softmax(_A      ,		axis=2     )

					# attend to values
					_UpperCAmelCase    :							Tuple									=					jnp.einsum("""b i j, b j d -> b i d"""      ,		_A      ,		_A     )

				_UpperCAmelCase    :							int									=					self.reshape_batch_dim_to_heads(_A     )
				_UpperCAmelCase    :							str									=					self.proj_attn(_A     )
				return self.dropout_layer(_A      ,		deterministic=_A     )






class     _UpperCAmelCase			(		nn.Module):
			__a :       int
			__a :       int
			__a :       int
			__a :       float									=							0.0
			__a :       bool									=							False
			__a :       jnp.dtype									=							jnp.floataa
			__a :       bool									=							False
			def 							__snake_case (		self     )    ->    Optional[Any]:






				'''simple docstring'''







				_UpperCAmelCase    :							Any									=					FlaxAttention(
				    self.dim      ,		self.n_heads      ,		self.d_head      ,		self.dropout      ,		self.use_memory_efficient_attention      ,		dtype=self.dtype     )
				# cross attention
				_UpperCAmelCase    :							Optional[int]									=					FlaxAttention(
				    self.dim      ,		self.n_heads      ,		self.d_head      ,		self.dropout      ,		self.use_memory_efficient_attention      ,		dtype=self.dtype     )
				_UpperCAmelCase    :							Optional[int]									=					FlaxFeedForward(dim=self.dim      ,		dropout=self.dropout      ,		dtype=self.dtype     )
				_UpperCAmelCase    :							str									=					nn.LayerNorm(epsilon=1e-5      ,		dtype=self.dtype     )
				_UpperCAmelCase    :							Any									=					nn.LayerNorm(epsilon=1e-5      ,		dtype=self.dtype     )
				_UpperCAmelCase    :							Any									=					nn.LayerNorm(epsilon=1e-5      ,		dtype=self.dtype     )
				_UpperCAmelCase    :							Dict									=					nn.Dropout(rate=self.dropout     )
			def __call__(		self      ,		_A      ,		_A      ,		_A=True     )    ->    Any:






				'''simple docstring'''







				_UpperCAmelCase    :							Any									=					hidden_states
				if self.only_cross_attention:
					_UpperCAmelCase    :							Tuple									=					self.attna(self.norma(_A     )      ,		_A      ,		deterministic=_A     )
				else:
					_UpperCAmelCase    :							str									=					self.attna(self.norma(_A     )      ,		deterministic=_A     )
				_UpperCAmelCase    :							Dict									=					hidden_states + residual

				# cross attention
				_UpperCAmelCase    :							List[Any]									=					hidden_states
				_UpperCAmelCase    :							List[Any]									=					self.attna(self.norma(_A     )      ,		_A      ,		deterministic=_A     )
				_UpperCAmelCase    :							int									=					hidden_states + residual

				# feed forward
				_UpperCAmelCase    :							str									=					hidden_states
				_UpperCAmelCase    :							Union[str, Any]									=					self.ff(self.norma(_A     )      ,		deterministic=_A     )
				_UpperCAmelCase    :							Union[str, Any]									=					hidden_states + residual

				return self.dropout_layer(_A      ,		deterministic=_A     )






class     _UpperCAmelCase			(		nn.Module):
			__a :       int
			__a :       int
			__a :       int
			__a :       int									=							1
			__a :       float									=							0.0
			__a :       bool									=							False
			__a :       bool									=							False
			__a :       jnp.dtype									=							jnp.floataa
			__a :       bool									=							False
			def 							__snake_case (		self     )    ->    str:






				'''simple docstring'''







				_UpperCAmelCase    :							Any									=					nn.GroupNorm(num_groups=32      ,		epsilon=1e-5     )

				_UpperCAmelCase    :							Optional[Any]									=					self.n_heads * self.d_head
				if self.use_linear_projection:
					_UpperCAmelCase    :							Optional[Any]									=					nn.Dense(_A      ,		dtype=self.dtype     )
				else:
					_UpperCAmelCase    :							List[Any]									=					nn.Conv(
					    _A      ,		kernel_size=(1, 1)      ,		strides=(1, 1)      ,		padding="""VALID"""      ,		dtype=self.dtype      ,		)

				_UpperCAmelCase    :							Optional[int]									=					[
				    FlaxBasicTransformerBlock(
				        _A      ,		self.n_heads      ,		self.d_head      ,		dropout=self.dropout      ,		only_cross_attention=self.only_cross_attention      ,		dtype=self.dtype      ,		use_memory_efficient_attention=self.use_memory_efficient_attention      ,		)
				    for _ in range(self.depth     )
				]

				if self.use_linear_projection:
					_UpperCAmelCase    :							Union[str, Any]									=					nn.Dense(_A      ,		dtype=self.dtype     )
				else:
					_UpperCAmelCase    :							Optional[Any]									=					nn.Conv(
					    _A      ,		kernel_size=(1, 1)      ,		strides=(1, 1)      ,		padding="""VALID"""      ,		dtype=self.dtype      ,		)

				_UpperCAmelCase    :							List[Any]									=					nn.Dropout(rate=self.dropout     )
			def __call__(		self      ,		_A      ,		_A      ,		_A=True     )    ->    Any:






				'''simple docstring'''







				_UpperCAmelCase						,   _UpperCAmelCase						,   _UpperCAmelCase						,   _UpperCAmelCase    :							int									=					hidden_states.shape
				_UpperCAmelCase    :							Tuple									=					hidden_states
				_UpperCAmelCase    :							Optional[Any]									=					self.norm(_A     )
				if self.use_linear_projection:
					_UpperCAmelCase    :							Dict									=					hidden_states.reshape(_A      ,		height * width      ,		_A     )
					_UpperCAmelCase    :							List[Any]									=					self.proj_in(_A     )
				else:
					_UpperCAmelCase    :							Union[str, Any]									=					self.proj_in(_A     )
					_UpperCAmelCase    :							List[Any]									=					hidden_states.reshape(_A      ,		height * width      ,		_A     )

				for transformer_block in self.transformer_blocks:
					_UpperCAmelCase    :							int									=					transformer_block(_A      ,		_A      ,		deterministic=_A     )

				if self.use_linear_projection:
					_UpperCAmelCase    :							Optional[int]									=					self.proj_out(_A     )
					_UpperCAmelCase    :							Dict									=					hidden_states.reshape(_A      ,		_A      ,		_A      ,		_A     )
				else:
					_UpperCAmelCase    :							str									=					hidden_states.reshape(_A      ,		_A      ,		_A      ,		_A     )
					_UpperCAmelCase    :							Union[str, Any]									=					self.proj_out(_A     )

				_UpperCAmelCase    :							List[Any]									=					hidden_states + residual
				return self.dropout_layer(_A      ,		deterministic=_A     )






class     _UpperCAmelCase			(		nn.Module):
			__a :       int
			__a :       float									=							0.0
			__a :       jnp.dtype									=							jnp.floataa
			def 							__snake_case (		self     )    ->    Any:






				'''simple docstring'''







				_UpperCAmelCase    :							Optional[int]									=					FlaxGEGLU(self.dim      ,		self.dropout      ,		self.dtype     )
				_UpperCAmelCase    :							Dict									=					nn.Dense(self.dim      ,		dtype=self.dtype     )
			def __call__(		self      ,		_A      ,		_A=True     )    ->    Union[str, Any]:






				'''simple docstring'''







				_UpperCAmelCase    :							Dict									=					self.net_a(_A      ,		deterministic=_A     )
				_UpperCAmelCase    :							Tuple									=					self.net_a(_A     )
				return hidden_states






class     _UpperCAmelCase			(		nn.Module):
			__a :       int
			__a :       float									=							0.0
			__a :       jnp.dtype									=							jnp.floataa
			def 							__snake_case (		self     )    ->    Tuple:






				'''simple docstring'''







				_UpperCAmelCase    :							Dict									=					self.dim * 4
				_UpperCAmelCase    :							Optional[Any]									=					nn.Dense(inner_dim * 2      ,		dtype=self.dtype     )
				_UpperCAmelCase    :							int									=					nn.Dropout(rate=self.dropout     )
			def __call__(		self      ,		_A      ,		_A=True     )    ->    int:






				'''simple docstring'''







				_UpperCAmelCase    :							Optional[Any]									=					self.proj(_A     )
				_UpperCAmelCase						,   _UpperCAmelCase    :							Optional[int]									=					jnp.split(_A      ,		2      ,		axis=2     )
				return self.dropout_layer(hidden_linear * nn.gelu(_A     )      ,		deterministic=_A     )








"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase__				:  Optional[int]													=  {'''configuration_t5''': ['''T5_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''T5Config''', '''T5OnnxConfig''']}

try:
						if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase__				:  Dict													=  ['''T5Tokenizer''']

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase__				:  List[str]													=  ['''T5TokenizerFast''']

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase__				:  Optional[Any]													=  [
						    '''T5_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''T5EncoderModel''',
						    '''T5ForConditionalGeneration''',
						    '''T5Model''',
						    '''T5PreTrainedModel''',
						    '''load_tf_weights_in_t5''',
						    '''T5ForQuestionAnswering''',
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase__				:  int													=  [
						    '''TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''TFT5EncoderModel''',
						    '''TFT5ForConditionalGeneration''',
						    '''TFT5Model''',
						    '''TFT5PreTrainedModel''',
						]

try:
						if not is_flax_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase__				:  Any													=  [
						    '''FlaxT5EncoderModel''',
						    '''FlaxT5ForConditionalGeneration''',
						    '''FlaxT5Model''',
						    '''FlaxT5PreTrainedModel''',
						]


if TYPE_CHECKING:
						from .configuration_ta import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, TaConfig, TaOnnxConfig

						try:
												if not is_sentencepiece_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_ta import TaTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_ta_fast import TaTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_ta import (
												    T5_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TaEncoderModel,
												    TaForConditionalGeneration,
												    TaForQuestionAnswering,
												    TaModel,
												    TaPreTrainedModel,
												    load_tf_weights_in_ta,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_ta import (
												    TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFTaEncoderModel,
												    TFTaForConditionalGeneration,
												    TFTaModel,
												    TFTaPreTrainedModel,
												)

						try:
												if not is_flax_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_flax_ta import (
												    FlaxTaEncoderModel,
												    FlaxTaForConditionalGeneration,
												    FlaxTaModel,
												    FlaxTaPreTrainedModel,
												)


else:
						import sys

						lowerCamelCase__				:  Dict													=  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





