
"""simple docstring"""






import random

class 				__a    :



       '''simple docstring'''







       def __init__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase = 0.01     ,						_lowerCamelCase = 1_000     ,						_lowerCamelCase = -1     ,						) ->       None:




          '''simple docstring'''
          __lowercase						      =					sample
          if len(self.sample					) == 0:
             raise ValueError("Sample data can not be empty"					)
          __lowercase						      =					target
          if len(self.target					) == 0:
             raise ValueError("Target data can not be empty"					)
          if len(self.sample					) != len(self.target					):
             raise ValueError("Sample data and Target data do not have matching lengths"					)
          __lowercase						      =					learning_rate
          __lowercase						      =					epoch_number
          __lowercase						      =					bias
          __lowercase						      =					len(_lowerCamelCase					)
          __lowercase						      =					len(sample[0]					)  # number of columns in dataset
          __lowercase						      =					[]
       def      SCREAMING_SNAKE_CASE	(    self					) ->       None:




          '''simple docstring'''
          for sample in self.sample:
             sample.insert(0     ,						self.bias					)

          for _ in range(self.col_sample					):
             self.weight.append(random.random()					)

          self.weight.insert(0     ,						self.bias					)

          __lowercase						      =					0

          while True:
             __lowercase						      =					False
             for i in range(self.number_sample					):
                __lowercase						      =					0
                for j in range(self.col_sample + 1					):
                   __lowercase						      =					u + self.weight[j] * self.sample[i][j]
                __lowercase						      =					self.sign(_lowerCamelCase					)
                if y != self.target[i]:
                   for j in range(self.col_sample + 1					):
                      __lowercase						      =					(
                          self.weight[j]
                          + self.learning_rate
                          * (self.target[i] - y)
                          * self.sample[i][j]
                      )
                   __lowercase						      =					True
                   # print('Epoch: \n',epoch_count)
             __lowercase						      =					epoch_count + 1
             # if you want control the epoch or just by error
             if not has_misclassified:
                print(("\nEpoch:\n", epoch_count)					)
                print("------------------------\n"					)
                # if epoch_count > self.epoch_number or not error:
                break
       def      SCREAMING_SNAKE_CASE	(    self     ,						_lowerCamelCase					) ->       None:




          '''simple docstring'''
          if len(self.sample					) == 0:
             raise ValueError("Sample data can not be empty"					)
          sample.insert(0     ,						self.bias					)
          __lowercase						      =					0
          for i in range(self.col_sample + 1					):
             __lowercase						      =					u + self.weight[i] * sample[i]

          __lowercase						      =					self.sign(_lowerCamelCase					)

          if y == -1:
             print(("Sample: ", sample)					)
             print("classification: P1"					)
          else:
             print(("Sample: ", sample)					)
             print("classification: P2"					)





       def      SCREAMING_SNAKE_CASE	(    self     ,						_lowerCamelCase					) ->       int:




          '''simple docstring'''
          return 1 if u >= 0 else -1





_lowercase							 =							[
    [-0.6_5_0_8, 0.1_0_9_7, 4.0_0_0_9],
    [-1.4_4_9_2, 0.8_8_9_6, 4.4_0_0_5],
    [2.0_8_5_0, 0.6_8_7_6, 1_2.0_7_1_0],
    [0.2_6_2_6, 1.1_4_7_6, 7.7_9_8_5],
    [0.6_4_1_8, 1.0_2_3_4, 7.0_4_2_7],
    [0.2_5_6_9, 0.6_7_3_0, 8.3_2_6_5],
    [1.1_1_5_5, 0.6_0_4_3, 7.4_4_4_6],
    [0.0_9_1_4, 0.3_3_9_9, 7.0_6_7_7],
    [0.0_1_2_1, 0.5_2_5_6, 4.6_3_1_6],
    [-0.0_4_2_9, 0.4_6_6_0, 5.4_3_2_3],
    [0.4_3_4_0, 0.6_8_7_0, 8.2_2_8_7],
    [0.2_7_3_5, 1.0_2_8_7, 7.1_9_3_4],
    [0.4_8_3_9, 0.4_8_5_1, 7.4_8_5_0],
    [0.4_0_8_9, -0.1_2_6_7, 5.5_0_1_9],
    [1.4_3_9_1, 0.1_6_1_4, 8.5_8_4_3],
    [-0.9_1_1_5, -0.1_9_7_3, 2.1_9_6_2],
    [0.3_6_5_4, 1.0_4_7_5, 7.4_8_5_8],
    [0.2_1_4_4, 0.7_5_1_5, 7.1_6_9_9],
    [0.2_0_1_3, 1.0_0_1_4, 6.5_4_8_9],
    [0.6_4_8_3, 0.2_1_8_3, 5.8_9_9_1],
    [-0.1_1_4_7, 0.2_2_4_2, 7.2_4_3_5],
    [-0.7_9_7_0, 0.8_7_9_5, 3.8_7_6_2],
    [-1.0_6_2_5, 0.6_3_6_6, 2.4_7_0_7],
    [0.5_3_0_7, 0.1_2_8_5, 5.6_8_8_3],
    [-1.2_2_0_0, 0.7_7_7_7, 1.7_2_5_2],
    [0.3_9_5_7, 0.1_0_7_6, 5.6_6_2_3],
    [-0.1_0_1_3, 0.5_9_8_9, 7.1_8_1_2],
    [2.4_4_8_2, 0.9_4_5_5, 1_1.2_0_9_5],
    [2.0_1_4_9, 0.6_1_9_2, 1_0.9_2_6_3],
    [0.2_0_1_2, 0.2_6_1_1, 5.4_6_3_1],
]

_lowercase							 =							[
    -1,
    -1,
    -1,
    1,
    1,
    -1,
    1,
    -1,
    1,
    1,
    -1,
    1,
    -1,
    -1,
    -1,
    -1,
    1,
    1,
    1,
    1,
    -1,
    1,
    1,
    1,
    1,
    -1,
    -1,
    1,
    -1,
    1,
]


if __name__ == "__main__":
     import doctest

     doctest.testmod()

     _lowercase							 =							Perceptron(
         sample=samples, target=target, learning_rate=0.0_1, epoch_number=1_000, bias=-1
     )
     network.training()
     print('''Finished training perceptron''')
     print('''Enter values to predict or q to exit''')
     while True:
          _lowercase							 =							[]
          for i in range(len(samples[0])):
               _lowercase							 =							input('''value: ''').strip()
               if user_input == "q":
                    break
               _lowercase							 =							float(user_input)
               sample.insert(i, observation)
          network.sort(sample)







"""simple docstring"""






import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp



def     lowerCAmelCase__ (		__magic_name__ ,      __magic_name__ ,      __magic_name__ ,      __magic_name__ ,      __magic_name__ = 4_0_9_6			)	->Dict:
			__lowercase      ,				__lowercase      ,				__lowercase						      =					key.shape[-3:]
			__lowercase						      =					value.shape[-1]
			__lowercase						      =					min(__magic_name__ ,      __magic_name__			)
			__lowercase						      =					query / jnp.sqrt(__magic_name__			)

			@functools.partial(jax.checkpoint ,      prevent_cse=__magic_name__			)
			def summarize_chunk(__magic_name__ ,      __magic_name__ ,      __magic_name__			):
						__lowercase						      =					jnp.einsum("...qhd,...khd->...qhk" ,      __magic_name__ ,      __magic_name__ ,      precision=__magic_name__			)

						__lowercase						      =					jnp.max(__magic_name__ ,      axis=-1 ,      keepdims=__magic_name__			)
						__lowercase						      =					jax.lax.stop_gradient(__magic_name__			)
						__lowercase						      =					jnp.exp(attn_weights - max_score			)

						__lowercase						      =					jnp.einsum("...vhf,...qhv->...qhf" ,      __magic_name__ ,      __magic_name__ ,      precision=__magic_name__			)
						__lowercase						      =					jnp.einsum("...qhk->...qh" ,      __magic_name__			)

						return (exp_values, exp_weights.sum(axis=-1			), max_score)

			def chunk_scanner(__magic_name__			):
						# julienne key array
						__lowercase						      =					jax.lax.dynamic_slice(
						    operand=__magic_name__ ,      start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0] ,      slice_sizes=list(key.shape[:-3]			) + [key_chunk_size, num_heads, k_features] ,      )

						# julienne value array
						__lowercase						      =					jax.lax.dynamic_slice(
						    operand=__magic_name__ ,      start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0] ,      slice_sizes=list(value.shape[:-3]			) + [key_chunk_size, num_heads, v_features] ,      )

						return summarize_chunk(__magic_name__ ,      __magic_name__ ,      __magic_name__			)

			__lowercase      ,				__lowercase      ,				__lowercase						      =					jax.lax.map(f=__magic_name__ ,      xs=jnp.arange(0 ,      __magic_name__ ,      __magic_name__			)			)

			__lowercase						      =					jnp.max(__magic_name__ ,      axis=0 ,      keepdims=__magic_name__			)
			__lowercase						      =					jnp.exp(chunk_max - global_max			)

			chunk_values *= jnp.expand_dims(__magic_name__ ,      axis=-1			)
			chunk_weights *= max_diffs

			__lowercase						      =					chunk_values.sum(axis=0			)
			__lowercase						      =					jnp.expand_dims(__magic_name__ ,      -1			).sum(axis=0			)

			return all_values / all_weights



def     lowerCAmelCase__ (		__magic_name__ ,      __magic_name__ ,      __magic_name__ ,      __magic_name__=jax.lax.Precision.HIGHEST ,      __magic_name__ = 1_0_2_4 ,      __magic_name__ = 4_0_9_6			)	->List[str]:
			__lowercase      ,				__lowercase      ,				__lowercase						      =					query.shape[-3:]

			def chunk_scanner(__magic_name__ ,      __magic_name__			):
						# julienne query array
						__lowercase						      =					jax.lax.dynamic_slice(
						    operand=__magic_name__ ,      start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0] ,      slice_sizes=list(query.shape[:-3]			) + [min(__magic_name__ ,      __magic_name__			), num_heads, q_features] ,      )

						return (
						    chunk_idx + query_chunk_size,  # unused ignore it
						    _query_chunk_attention(
						        query=__magic_name__ ,      key=__magic_name__ ,      value=__magic_name__ ,      precision=__magic_name__ ,      key_chunk_size=__magic_name__			),
						)

			__lowercase      ,				__lowercase						      =					jax.lax.scan(
			    f=__magic_name__ ,      init=0 ,      xs=__magic_name__ ,      length=math.ceil(num_q / query_chunk_size			)			)

			return jnp.concatenate(__magic_name__ ,      axis=-3			)  # fuse the chunked result back

class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: int					      =      8
							_lowerCamelCase				: int					      =      64
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							def      SCREAMING_SNAKE_CASE	(    self					) ->       Any:




										'''simple docstring'''
										__lowercase						      =					self.dim_head * self.heads
										__lowercase						      =					self.dim_head**-0.5

										# Weights were exported with old names {to_q, to_k, to_v, to_out}
										__lowercase						      =					nn.Dense(_lowerCamelCase     ,						use_bias=_lowerCamelCase     ,						dtype=self.dtype     ,						name="to_q"					)
										__lowercase						      =					nn.Dense(_lowerCamelCase     ,						use_bias=_lowerCamelCase     ,						dtype=self.dtype     ,						name="to_k"					)
										__lowercase						      =					nn.Dense(_lowerCamelCase     ,						use_bias=_lowerCamelCase     ,						dtype=self.dtype     ,						name="to_v"					)

										__lowercase						      =					nn.Dense(self.query_dim     ,						dtype=self.dtype     ,						name="to_out_0"					)
										__lowercase						      =					nn.Dropout(rate=self.dropout					)
							def      SCREAMING_SNAKE_CASE	(    self     ,						_lowerCamelCase					) ->       Optional[int]:




										'''simple docstring'''
										__lowercase      ,				__lowercase      ,				__lowercase						      =					tensor.shape
										__lowercase						      =					self.heads
										__lowercase						      =					tensor.reshape(_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						dim // head_size					)
										__lowercase						      =					jnp.transpose(_lowerCamelCase     ,						(0, 2, 1, 3)					)
										__lowercase						      =					tensor.reshape(batch_size * head_size     ,						_lowerCamelCase     ,						dim // head_size					)
										return tensor
							def      SCREAMING_SNAKE_CASE	(    self     ,						_lowerCamelCase					) ->       str:




										'''simple docstring'''
										__lowercase      ,				__lowercase      ,				__lowercase						      =					tensor.shape
										__lowercase						      =					self.heads
										__lowercase						      =					tensor.reshape(batch_size // head_size     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase					)
										__lowercase						      =					jnp.transpose(_lowerCamelCase     ,						(0, 2, 1, 3)					)
										__lowercase						      =					tensor.reshape(batch_size // head_size     ,						_lowerCamelCase     ,						dim * head_size					)
										return tensor





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase=None     ,						_lowerCamelCase=True					) ->       List[str]:




										'''simple docstring'''
										__lowercase						      =					hidden_states if context is None else context

										__lowercase						      =					self.query(_lowerCamelCase					)
										__lowercase						      =					self.key(_lowerCamelCase					)
										__lowercase						      =					self.value(_lowerCamelCase					)

										__lowercase						      =					self.reshape_heads_to_batch_dim(_lowerCamelCase					)
										__lowercase						      =					self.reshape_heads_to_batch_dim(_lowerCamelCase					)
										__lowercase						      =					self.reshape_heads_to_batch_dim(_lowerCamelCase					)

										if self.use_memory_efficient_attention:
													__lowercase						      =					query_states.transpose(1     ,						0     ,						2					)
													__lowercase						      =					key_states.transpose(1     ,						0     ,						2					)
													__lowercase						      =					value_states.transpose(1     ,						0     ,						2					)

													# this if statement create a chunk size for each layer of the unet
													# the chunk size is equal to the query_length dimension of the deepest layer of the unet

													__lowercase						      =					query_states.shape[-3]
													if flatten_latent_dim % 64 == 0:
																__lowercase						      =					int(flatten_latent_dim / 64					)
													elif flatten_latent_dim % 16 == 0:
																__lowercase						      =					int(flatten_latent_dim / 16					)
													elif flatten_latent_dim % 4 == 0:
																__lowercase						      =					int(flatten_latent_dim / 4					)
													else:
																__lowercase						      =					int(_lowerCamelCase					)

													__lowercase						      =					jax_memory_efficient_attention(
													    _lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						query_chunk_size=_lowerCamelCase     ,						key_chunk_size=4_096 * 4					)

													__lowercase						      =					hidden_states.transpose(1     ,						0     ,						2					)
										else:
													# compute attentions
													__lowercase						      =					jnp.einsum("b i d, b j d->b i j"     ,						_lowerCamelCase     ,						_lowerCamelCase					)
													__lowercase						      =					attention_scores * self.scale
													__lowercase						      =					nn.softmax(_lowerCamelCase     ,						axis=2					)

													# attend to values
													__lowercase						      =					jnp.einsum("b i j, b j d -> b i d"     ,						_lowerCamelCase     ,						_lowerCamelCase					)

										__lowercase						      =					self.reshape_batch_dim_to_heads(_lowerCamelCase					)
										__lowercase						      =					self.proj_attn(_lowerCamelCase					)
										return self.dropout_layer(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							_lowerCamelCase				: bool					      =      False
							def      SCREAMING_SNAKE_CASE	(    self					) ->       str:




										'''simple docstring'''
										# self attention (or cross_attention if only_cross_attention is True)
										__lowercase						      =					FlaxAttention(
										    self.dim     ,						self.n_heads     ,						self.d_head     ,						self.dropout     ,						self.use_memory_efficient_attention     ,						dtype=self.dtype					)
										# cross attention
										__lowercase						      =					FlaxAttention(
										    self.dim     ,						self.n_heads     ,						self.d_head     ,						self.dropout     ,						self.use_memory_efficient_attention     ,						dtype=self.dtype					)
										__lowercase						      =					FlaxFeedForward(dim=self.dim     ,						dropout=self.dropout     ,						dtype=self.dtype					)
										__lowercase						      =					nn.LayerNorm(epsilon=1e-5     ,						dtype=self.dtype					)
										__lowercase						      =					nn.LayerNorm(epsilon=1e-5     ,						dtype=self.dtype					)
										__lowercase						      =					nn.LayerNorm(epsilon=1e-5     ,						dtype=self.dtype					)
										__lowercase						      =					nn.Dropout(rate=self.dropout					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       Optional[int]:




										'''simple docstring'''
										# self attention
										__lowercase						      =					hidden_states
										if self.only_cross_attention:
													__lowercase						      =					self.attna(self.norma(_lowerCamelCase					)     ,						_lowerCamelCase     ,						deterministic=_lowerCamelCase					)
										else:
													__lowercase						      =					self.attna(self.norma(_lowerCamelCase					)     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					hidden_states + residual

										# cross attention
										__lowercase						      =					hidden_states
										__lowercase						      =					self.attna(self.norma(_lowerCamelCase					)     ,						_lowerCamelCase     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					hidden_states + residual

										# feed forward
										__lowercase						      =					hidden_states
										__lowercase						      =					self.ff(self.norma(_lowerCamelCase					)     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					hidden_states + residual

										return self.dropout_layer(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: int					      =      1
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							_lowerCamelCase				: bool					      =      False
							def      SCREAMING_SNAKE_CASE	(    self					) ->       int:




										'''simple docstring'''
										__lowercase						      =					nn.GroupNorm(num_groups=32     ,						epsilon=1e-5					)

										__lowercase						      =					self.n_heads * self.d_head
										if self.use_linear_projection:
													__lowercase						      =					nn.Dense(_lowerCamelCase     ,						dtype=self.dtype					)
										else:
													__lowercase						      =					nn.Conv(
													    _lowerCamelCase     ,						kernel_size=(1, 1)     ,						strides=(1, 1)     ,						padding="VALID"     ,						dtype=self.dtype     ,						)

										__lowercase						      =					[
										    FlaxBasicTransformerBlock(
										        _lowerCamelCase     ,						self.n_heads     ,						self.d_head     ,						dropout=self.dropout     ,						only_cross_attention=self.only_cross_attention     ,						dtype=self.dtype     ,						use_memory_efficient_attention=self.use_memory_efficient_attention     ,						)
										    for _ in range(self.depth					)
										]

										if self.use_linear_projection:
													__lowercase						      =					nn.Dense(_lowerCamelCase     ,						dtype=self.dtype					)
										else:
													__lowercase						      =					nn.Conv(
													    _lowerCamelCase     ,						kernel_size=(1, 1)     ,						strides=(1, 1)     ,						padding="VALID"     ,						dtype=self.dtype     ,						)

										__lowercase						      =					nn.Dropout(rate=self.dropout					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       int:




										'''simple docstring'''
										__lowercase      ,				__lowercase      ,				__lowercase      ,				__lowercase						      =					hidden_states.shape
										__lowercase						      =					hidden_states
										__lowercase						      =					self.norm(_lowerCamelCase					)
										if self.use_linear_projection:
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						height * width     ,						_lowerCamelCase					)
													__lowercase						      =					self.proj_in(_lowerCamelCase					)
										else:
													__lowercase						      =					self.proj_in(_lowerCamelCase					)
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						height * width     ,						_lowerCamelCase					)

										for transformer_block in self.transformer_blocks:
													__lowercase						      =					transformer_block(_lowerCamelCase     ,						_lowerCamelCase     ,						deterministic=_lowerCamelCase					)

										if self.use_linear_projection:
													__lowercase						      =					self.proj_out(_lowerCamelCase					)
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase					)
										else:
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase					)
													__lowercase						      =					self.proj_out(_lowerCamelCase					)

										__lowercase						      =					hidden_states + residual
										return self.dropout_layer(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							def      SCREAMING_SNAKE_CASE	(    self					) ->       str:




										'''simple docstring'''
										# The second linear layer needs to be called
										# net_2 for now to match the index of the Sequential layer
										__lowercase						      =					FlaxGEGLU(self.dim     ,						self.dropout     ,						self.dtype					)
										__lowercase						      =					nn.Dense(self.dim     ,						dtype=self.dtype					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       int:




										'''simple docstring'''
										__lowercase						      =					self.net_a(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					self.net_a(_lowerCamelCase					)
										return hidden_states





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							def      SCREAMING_SNAKE_CASE	(    self					) ->       List[str]:




										'''simple docstring'''
										__lowercase						      =					self.dim * 4
										__lowercase						      =					nn.Dense(inner_dim * 2     ,						dtype=self.dtype					)
										__lowercase						      =					nn.Dropout(rate=self.dropout					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       Union[str, Any]:




										'''simple docstring'''
										__lowercase						      =					self.proj(_lowerCamelCase					)
										__lowercase      ,				__lowercase						      =					jnp.split(_lowerCamelCase     ,						2     ,						axis=2					)
										return self.dropout_layer(hidden_linear * nn.gelu(_lowerCamelCase					)     ,						deterministic=_lowerCamelCase					)





