import json
import os
import shutil
import sys
import tempfile
import unittest
from unittest import TestCase
from unittest.mock import patch

import faiss
import numpy as np
from datasets import Dataset

from transformers import BartConfig, BartTokenizer, DPRConfig, DPRQuestionEncoderTokenizer, RagConfig
from transformers.file_utils import is_datasets_available, is_faiss_available, is_psutil_available, is_torch_available
from transformers.integrations import is_ray_available
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_ray


sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # noqa: E402 # isort:skip

if is_torch_available():
			from distributed_pytorch_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
else:
			_SCREAMING_SNAKE_CASE:						List[str]    	=       None

if is_ray_available():
			import ray  # noqa: E402 # isort:skip
			from distributed_ray_retriever import RagRayDistributedRetriever, RayRetriever  # noqa: E402 # isort:skip
else:
			_SCREAMING_SNAKE_CASE:						Optional[int]    	=       None
			_SCREAMING_SNAKE_CASE:						Dict    	=       None
			_SCREAMING_SNAKE_CASE:						Union[str, Any]    	=       None






def    UpperCAmelCase__(						lowerCamelCase_  ):



			"""simple docstring"""





			if not (is_datasets_available() and is_faiss_available() and is_psutil_available()):
						_snake_case              =      unittest.skip("test requires Datasets, Faiss, psutil"  )(lowerCamelCase_  )
			return test_case







@require_distributed_retrieval
class 	_SCREAMING_SNAKE_CASE							(     _lowercase							):






		"""simple docstring"""







		def 				_a				(			self    :				Any   ):






					"""simple docstring"""
					_snake_case              =      tempfile.mkdtemp()
					_snake_case              =      8

					# DPR tok
					_snake_case              =      [
					    "[UNK]",
					    "[CLS]",
					    "[SEP]",
					    "[PAD]",
					    "[MASK]",
					    "want",
					    "##want",
					    "##ed",
					    "wa",
					    "un",
					    "runn",
					    "##ing",
					    ",",
					    "low",
					    "lowest",
					]
					_snake_case              =      os.path.join(self.tmpdirname  ,					"dpr_tokenizer"   )
					os.makedirs(__SCREAMING_SNAKE_CASE  ,					exist_ok=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      os.path.join(__SCREAMING_SNAKE_CASE  ,					DPR_VOCAB_FILES_NAMES["vocab_file"]   )
					with open(self.vocab_file  ,					"w"  ,					encoding="utf-8"   ) as vocab_writer:
								vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )

					# BART tok
					_snake_case              =      [
					    "l",
					    "o",
					    "w",
					    "e",
					    "r",
					    "s",
					    "t",
					    "i",
					    "d",
					    "n",
					    "\u0120",
					    "\u0120l",
					    "\u0120n",
					    "\u0120lo",
					    "\u0120low",
					    "er",
					    "\u0120lowest",
					    "\u0120newer",
					    "\u0120wider",
					    "<unk>",
					]
					_snake_case              =      dict(zip(__SCREAMING_SNAKE_CASE  ,					range(len(__SCREAMING_SNAKE_CASE   )   )   )   )
					_snake_case              =      ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
					_snake_case              =      {"unk_token": "<unk>"}

					_snake_case              =      os.path.join(self.tmpdirname  ,					"bart_tokenizer"   )
					os.makedirs(__SCREAMING_SNAKE_CASE  ,					exist_ok=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      os.path.join(__SCREAMING_SNAKE_CASE  ,					BART_VOCAB_FILES_NAMES["vocab_file"]   )
					_snake_case              =      os.path.join(__SCREAMING_SNAKE_CASE  ,					BART_VOCAB_FILES_NAMES["merges_file"]   )
					with open(self.vocab_file  ,					"w"  ,					encoding="utf-8"   ) as fp:
								fp.write(json.dumps(__SCREAMING_SNAKE_CASE   ) + "\n"   )
					with open(self.merges_file  ,					"w"  ,					encoding="utf-8"   ) as fp:
								fp.write("\n".join(__SCREAMING_SNAKE_CASE   )   )



		def 				_a				(			self    :				Union[str, Any]   ):






					"""simple docstring"""
					return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,					"dpr_tokenizer"   )   )



		def 				_a				(			self    :				Union[str, Any]   ):






					"""simple docstring"""
					return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,					"bart_tokenizer"   )   )



		def 				_a				(			self    :				List[Any]   ):






					"""simple docstring"""
					shutil.rmtree(self.tmpdirname   )



		def 				_a				(			self    :				Dict   ):






					"""simple docstring"""
					_snake_case              =      Dataset.from_dict(
					    {
					        "id": ["0", "1"],
					        "text": ["foo", "bar"],
					        "title": ["Foo", "Bar"],
					        "embeddings": [np.ones(self.retrieval_vector_size   ), 2 * np.ones(self.retrieval_vector_size   )],
					    }   )
					dataset.add_faiss_index("embeddings"  ,					string_factory="Flat"  ,					metric_type=faiss.METRIC_INNER_PRODUCT   )
					return dataset



		def 				_a				(			self    :				int  ,					__SCREAMING_SNAKE_CASE    :				bool  ,					__SCREAMING_SNAKE_CASE    :				Tuple=12_345   ):






					"""simple docstring"""
					_snake_case              =      self.get_dummy_dataset()
					_snake_case              =      RagConfig(
					    retrieval_vector_size=self.retrieval_vector_size  ,					question_encoder=DPRConfig().to_dict()  ,					generator=BartConfig().to_dict()  ,					)
					with patch("transformers.models.rag.retrieval_rag.load_dataset"   ) as mock_load_dataset:
								_snake_case              =      dataset
								_snake_case              =      RagPyTorchDistributedRetriever(
								    __SCREAMING_SNAKE_CASE  ,					question_encoder_tokenizer=self.get_dpr_tokenizer()  ,					generator_tokenizer=self.get_bart_tokenizer()  ,					)
								if init_retrieval:
											retriever.init_retrieval(__SCREAMING_SNAKE_CASE   )
					return retriever



		def 				_a				(			self    :				Tuple  ,					__SCREAMING_SNAKE_CASE    :				bool   ):






					"""simple docstring"""
					# Have to run in local mode because sys.path modifications at top of
					# file are not propogated to remote workers.
					# https://stackoverflow.com/questions/54338013/parallel-import-a-python-file-from-sibling-folder
					ray.init(local_mode=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      RagConfig(
					    retrieval_vector_size=self.retrieval_vector_size  ,					question_encoder=DPRConfig().to_dict()  ,					generator=BartConfig().to_dict()  ,					)
					_snake_case              =      ray.remote(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      [remote_cls.remote() for _ in range(1   )]
					with patch("transformers.models.rag.retrieval_rag.load_dataset"   ) as mock_load_dataset:
								_snake_case              =      self.get_dummy_dataset()
								_snake_case              =      RagRayDistributedRetriever(
								    __SCREAMING_SNAKE_CASE  ,					question_encoder_tokenizer=self.get_dpr_tokenizer()  ,					generator_tokenizer=self.get_bart_tokenizer()  ,					retrieval_workers=__SCREAMING_SNAKE_CASE  ,					)
								if init_retrieval:
											retriever.init_retrieval()
					return retriever



		def 				_a				(			self    :				Dict  ,					__SCREAMING_SNAKE_CASE    :				bool  ,					__SCREAMING_SNAKE_CASE    :				bool  ,					__SCREAMING_SNAKE_CASE    :				List[str]=12_345   ):






					"""simple docstring"""
					_snake_case              =      self.get_dummy_dataset()
					_snake_case              =      RagConfig(
					    retrieval_vector_size=self.retrieval_vector_size  ,					question_encoder=DPRConfig().to_dict()  ,					generator=BartConfig().to_dict()  ,					index_name="custom"  ,					)
					if from_disk:
								_snake_case              =      os.path.join(self.tmpdirname  ,					"dataset"   )
								_snake_case              =      os.path.join(self.tmpdirname  ,					"index.faiss"   )
								dataset.get_index("embeddings"   ).save(os.path.join(self.tmpdirname  ,					"index.faiss"   )   )
								dataset.drop_index("embeddings"   )
								dataset.save_to_disk(os.path.join(self.tmpdirname  ,					"dataset"   )   )
								del dataset
								_snake_case              =      RagPyTorchDistributedRetriever(
								    __SCREAMING_SNAKE_CASE  ,					question_encoder_tokenizer=self.get_dpr_tokenizer()  ,					generator_tokenizer=self.get_bart_tokenizer()  ,					)
					else:
								_snake_case              =      RagPyTorchDistributedRetriever(
								    __SCREAMING_SNAKE_CASE  ,					question_encoder_tokenizer=self.get_dpr_tokenizer()  ,					generator_tokenizer=self.get_bart_tokenizer()  ,					index=CustomHFIndex(config.retrieval_vector_size  ,					__SCREAMING_SNAKE_CASE   )  ,					)
					if init_retrieval:
								retriever.init_retrieval(__SCREAMING_SNAKE_CASE   )
					return retriever



		def 				_a				(			self    :				Dict  ,					__SCREAMING_SNAKE_CASE    :				bool  ,					__SCREAMING_SNAKE_CASE    :				bool   ):






					"""simple docstring"""
					# Have to run in local mode because sys.path modifications at top of
					# file are not propogated to remote workers.
					# https://stackoverflow.com/questions/54338013/parallel-import-a-python-file-from-sibling-folder
					ray.init(local_mode=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.get_dummy_dataset()
					_snake_case              =      RagConfig(
					    retrieval_vector_size=self.retrieval_vector_size  ,					question_encoder=DPRConfig().to_dict()  ,					generator=BartConfig().to_dict()  ,					index_name="custom"  ,					)
					_snake_case              =      ray.remote(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      [remote_cls.remote() for _ in range(1   )]
					if from_disk:
								_snake_case              =      os.path.join(self.tmpdirname  ,					"dataset"   )
								_snake_case              =      os.path.join(self.tmpdirname  ,					"index.faiss"   )
								dataset.get_index("embeddings"   ).save(os.path.join(self.tmpdirname  ,					"index.faiss"   )   )
								dataset.drop_index("embeddings"   )
								dataset.save_to_disk(os.path.join(self.tmpdirname  ,					"dataset"   )   )
								del dataset
								_snake_case              =      RagRayDistributedRetriever(
								    __SCREAMING_SNAKE_CASE  ,					question_encoder_tokenizer=self.get_dpr_tokenizer()  ,					generator_tokenizer=self.get_bart_tokenizer()  ,					retrieval_workers=__SCREAMING_SNAKE_CASE  ,					index=CustomHFIndex.load_from_disk(
								        vector_size=config.retrieval_vector_size  ,					dataset_path=config.passages_path  ,					index_path=config.index_path  ,					)  ,					)
					else:
								_snake_case              =      RagRayDistributedRetriever(
								    __SCREAMING_SNAKE_CASE  ,					question_encoder_tokenizer=self.get_dpr_tokenizer()  ,					generator_tokenizer=self.get_bart_tokenizer()  ,					retrieval_workers=__SCREAMING_SNAKE_CASE  ,					index=CustomHFIndex(config.retrieval_vector_size  ,					__SCREAMING_SNAKE_CASE   )  ,					)
					if init_retrieval:
								retriever.init_retrieval()
					return retriever



		def 				_a				(			self    :				Optional[Any]  ,					__SCREAMING_SNAKE_CASE    :				RagRetriever  ,					__SCREAMING_SNAKE_CASE    :				np.array  ,					__SCREAMING_SNAKE_CASE    :				int   ):






					"""simple docstring"""
					_snake_case       , _snake_case       , _snake_case              =      retriever.retrieve(__SCREAMING_SNAKE_CASE  ,					n_docs=__SCREAMING_SNAKE_CASE   )
					self.assertEqual(retrieved_doc_embeds.shape  ,					(2, n_docs, self.retrieval_vector_size)   )
					self.assertEqual(len(__SCREAMING_SNAKE_CASE   )  ,					2   )
					self.assertEqual(sorted(doc_dicts[0]   )  ,					["embeddings", "id", "text", "title"]   )
					self.assertEqual(len(doc_dicts[0]["id"]   )  ,					__SCREAMING_SNAKE_CASE   )
					self.assertEqual(doc_dicts[0]["id"][0]  ,					"1"   )  # max inner product is reached with second doc
					self.assertEqual(doc_dicts[1]["id"][0]  ,					"0"   )  # max inner product is reached with first doc
					self.assertListEqual(doc_ids.tolist()  ,					[[1], [0]]   )



		def 				_a				(			self    :				Tuple   ):






					"""simple docstring"""
					_snake_case              =      1
					_snake_case              =      np.array(
					    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]  ,					dtype=np.floataa   )

					self.distributed_retriever_check(
					    self.get_dummy_pytorch_distributed_retriever(init_retrieval=__SCREAMING_SNAKE_CASE   )  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )



		def 				_a				(			self    :				str   ):






					"""simple docstring"""
					_snake_case              =      1
					_snake_case              =      np.array(
					    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]  ,					dtype=np.floataa   )

					self.distributed_retriever_check(
					    self.get_dummy_custom_hf_index_pytorch_retriever(init_retrieval=__SCREAMING_SNAKE_CASE  ,					from_disk=__SCREAMING_SNAKE_CASE   )  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					)



		def 				_a				(			self    :				Optional[Any]   ):






					"""simple docstring"""
					_snake_case              =      1
					_snake_case              =      np.array(
					    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]  ,					dtype=np.floataa   )

					self.distributed_retriever_check(
					    self.get_dummy_custom_hf_index_pytorch_retriever(init_retrieval=__SCREAMING_SNAKE_CASE  ,					from_disk=__SCREAMING_SNAKE_CASE   )  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					)



		@require_ray
		def 				_a				(			self    :				str   ):






					"""simple docstring"""
					_snake_case              =      1
					_snake_case              =      np.array(
					    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]  ,					dtype=np.floataa   )

					self.distributed_retriever_check(
					    self.get_dummy_ray_distributed_retriever(init_retrieval=__SCREAMING_SNAKE_CASE   )  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )
					ray.shutdown()



		@require_ray
		def 				_a				(			self    :				Any   ):






					"""simple docstring"""
					_snake_case              =      1
					_snake_case              =      np.array(
					    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]  ,					dtype=np.floataa   )
					with self.assertRaises(__SCREAMING_SNAKE_CASE   ):
								self.distributed_retriever_check(
								    self.get_dummy_custom_hf_index_ray_retriever(init_retrieval=__SCREAMING_SNAKE_CASE  ,					from_disk=__SCREAMING_SNAKE_CASE   )  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					)
					ray.shutdown()







		@require_ray
		def 				_a				(			self    :				List[str]   ):






					"""simple docstring"""
					_snake_case              =      1
					_snake_case              =      np.array(
					    [np.ones(self.retrieval_vector_size   ), -np.ones(self.retrieval_vector_size   )]  ,					dtype=np.floataa   )

					self.distributed_retriever_check(
					    self.get_dummy_custom_hf_index_ray_retriever(init_retrieval=__SCREAMING_SNAKE_CASE  ,					from_disk=__SCREAMING_SNAKE_CASE   )  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )
					ray.shutdown()


import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp






def    UpperCAmelCase__(						lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_ = 4096  ):



			"""simple docstring"""





			_snake_case       , _snake_case       , _snake_case              =      key.shape[-3:]
			_snake_case              =      value.shape[-1]
			_snake_case              =      min(lowerCamelCase_							,	lowerCamelCase_  )
			_snake_case              =      query / jnp.sqrt(lowerCamelCase_  )

			@functools.partial(jax.checkpoint							,	prevent_cse=lowerCamelCase_  )
			def summarize_chunk(lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_  ):
						_snake_case              =      jnp.einsum("...qhd,...khd->...qhk"							,	lowerCamelCase_							,	lowerCamelCase_							,	precision=lowerCamelCase_  )

						_snake_case              =      jnp.max(lowerCamelCase_							,	axis=-1							,	keepdims=lowerCamelCase_  )
						_snake_case              =      jax.lax.stop_gradient(lowerCamelCase_  )
						_snake_case              =      jnp.exp(attn_weights - max_score  )

						_snake_case              =      jnp.einsum("...vhf,...qhv->...qhf"							,	lowerCamelCase_							,	lowerCamelCase_							,	precision=lowerCamelCase_  )
						_snake_case              =      jnp.einsum("...qhk->...qh"							,	lowerCamelCase_  )

						return (exp_values, exp_weights.sum(axis=-1  ), max_score)

			def chunk_scanner(lowerCamelCase_  ):
						# julienne key array
						_snake_case              =      jax.lax.dynamic_slice(
						    operand=lowerCamelCase_							,	start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]							,	slice_sizes=list(key.shape[:-3]  ) + [key_chunk_size, num_heads, k_features]							,	)

						# julienne value array
						_snake_case              =      jax.lax.dynamic_slice(
						    operand=lowerCamelCase_							,	start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]							,	slice_sizes=list(value.shape[:-3]  ) + [key_chunk_size, num_heads, v_features]							,	)

						return summarize_chunk(lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_  )

			_snake_case       , _snake_case       , _snake_case              =      jax.lax.map(f=lowerCamelCase_							,	xs=jnp.arange(0							,	lowerCamelCase_							,	lowerCamelCase_  )  )

			_snake_case              =      jnp.max(lowerCamelCase_							,	axis=0							,	keepdims=lowerCamelCase_  )
			_snake_case              =      jnp.exp(chunk_max - global_max  )

			chunk_values *= jnp.expand_dims(lowerCamelCase_							,	axis=-1  )
			chunk_weights *= max_diffs

			_snake_case              =      chunk_values.sum(axis=0  )
			_snake_case              =      jnp.expand_dims(lowerCamelCase_							,	-1  ).sum(axis=0  )

			return all_values / all_weights






def    UpperCAmelCase__(						lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_							,	lowerCamelCase_=jax.lax.Precision.HIGHEST							,	lowerCamelCase_ = 1024							,	lowerCamelCase_ = 4096  ):



			"""simple docstring"""





			_snake_case       , _snake_case       , _snake_case              =      query.shape[-3:]

			def chunk_scanner(lowerCamelCase_							,	lowerCamelCase_  ):
						# julienne query array
						_snake_case              =      jax.lax.dynamic_slice(
						    operand=lowerCamelCase_							,	start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]							,	slice_sizes=list(query.shape[:-3]  ) + [min(lowerCamelCase_							,	lowerCamelCase_  ), num_heads, q_features]							,	)

						return (
						    chunk_idx + query_chunk_size,  # unused ignore it
						    _query_chunk_attention(
						        query=lowerCamelCase_							,	key=lowerCamelCase_							,	value=lowerCamelCase_							,	precision=lowerCamelCase_							,	key_chunk_size=lowerCamelCase_  ),
						)

			_snake_case       , _snake_case              =      jax.lax.scan(
			    f=lowerCamelCase_							,	init=0							,	xs=lowerCamelCase_							,	length=math.ceil(num_q / query_chunk_size  )  )

			return jnp.concatenate(lowerCamelCase_							,	axis=-3  )  # fuse the chunked result back

class 	_SCREAMING_SNAKE_CASE							(     nn.Module							):






		"""simple docstring"""







		A_						 =      42
		A_						 =      8
		A_						 =      64
		A_						 =      0.0
		A_						 =      False
		A_						 =      jnp.floataa



		def 				_a				(			self    :				Union[str, Any]   ):






					"""simple docstring"""
					_snake_case              =      self.dim_head * self.heads
					_snake_case              =      self.dim_head**-0.5

					# Weights were exported with old names {to_q, to_k, to_v, to_out}
					_snake_case              =      nn.Dense(__SCREAMING_SNAKE_CASE  ,					use_bias=__SCREAMING_SNAKE_CASE  ,					dtype=self.dtype  ,					name="to_q"   )
					_snake_case              =      nn.Dense(__SCREAMING_SNAKE_CASE  ,					use_bias=__SCREAMING_SNAKE_CASE  ,					dtype=self.dtype  ,					name="to_k"   )
					_snake_case              =      nn.Dense(__SCREAMING_SNAKE_CASE  ,					use_bias=__SCREAMING_SNAKE_CASE  ,					dtype=self.dtype  ,					name="to_v"   )

					_snake_case              =      nn.Dense(self.query_dim  ,					dtype=self.dtype  ,					name="to_out_0"   )
					_snake_case              =      nn.Dropout(rate=self.dropout   )



		def 				_a				(			self    :				Optional[int]  ,					__SCREAMING_SNAKE_CASE    :				Any   ):






					"""simple docstring"""
					_snake_case       , _snake_case       , _snake_case              =      tensor.shape
					_snake_case              =      self.heads
					_snake_case              =      tensor.reshape(__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					dim // head_size   )
					_snake_case              =      jnp.transpose(__SCREAMING_SNAKE_CASE  ,					(0, 2, 1, 3)   )
					_snake_case              =      tensor.reshape(batch_size * head_size  ,					__SCREAMING_SNAKE_CASE  ,					dim // head_size   )
					return tensor



		def 				_a				(			self    :				str  ,					__SCREAMING_SNAKE_CASE    :				Tuple   ):






					"""simple docstring"""
					_snake_case       , _snake_case       , _snake_case              =      tensor.shape
					_snake_case              =      self.heads
					_snake_case              =      tensor.reshape(batch_size // head_size  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )
					_snake_case              =      jnp.transpose(__SCREAMING_SNAKE_CASE  ,					(0, 2, 1, 3)   )
					_snake_case              =      tensor.reshape(batch_size // head_size  ,					__SCREAMING_SNAKE_CASE  ,					dim * head_size   )
					return tensor







		def __call__(			self    :				int  ,					__SCREAMING_SNAKE_CASE    :				int  ,					__SCREAMING_SNAKE_CASE    :				Tuple=None  ,					__SCREAMING_SNAKE_CASE    :				List[Any]=True   ):






					"""simple docstring"""
					_snake_case              =      hidden_states if context is None else context

					_snake_case              =      self.query(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.key(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.value(__SCREAMING_SNAKE_CASE   )

					_snake_case              =      self.reshape_heads_to_batch_dim(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.reshape_heads_to_batch_dim(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.reshape_heads_to_batch_dim(__SCREAMING_SNAKE_CASE   )

					if self.use_memory_efficient_attention:
								_snake_case              =      query_states.transpose(1  ,					0  ,					2   )
								_snake_case              =      key_states.transpose(1  ,					0  ,					2   )
								_snake_case              =      value_states.transpose(1  ,					0  ,					2   )

								# this if statement create a chunk size for each layer of the unet
								# the chunk size is equal to the query_length dimension of the deepest layer of the unet

								_snake_case              =      query_states.shape[-3]
								if flatten_latent_dim % 64 == 0:
											_snake_case              =      int(flatten_latent_dim / 64   )
								elif flatten_latent_dim % 16 == 0:
											_snake_case              =      int(flatten_latent_dim / 16   )
								elif flatten_latent_dim % 4 == 0:
											_snake_case              =      int(flatten_latent_dim / 4   )
								else:
											_snake_case              =      int(__SCREAMING_SNAKE_CASE   )

								_snake_case              =      jax_memory_efficient_attention(
								    __SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					query_chunk_size=__SCREAMING_SNAKE_CASE  ,					key_chunk_size=4_096 * 4   )

								_snake_case              =      hidden_states.transpose(1  ,					0  ,					2   )
					else:
								# compute attentions
								_snake_case              =      jnp.einsum("b i d, b j d->b i j"  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )
								_snake_case              =      attention_scores * self.scale
								_snake_case              =      nn.softmax(__SCREAMING_SNAKE_CASE  ,					axis=2   )

								# attend to values
								_snake_case              =      jnp.einsum("b i j, b j d -> b i d"  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )

					_snake_case              =      self.reshape_batch_dim_to_heads(__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.proj_attn(__SCREAMING_SNAKE_CASE   )
					return self.dropout_layer(__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )

class 	_SCREAMING_SNAKE_CASE							(     nn.Module							):






		"""simple docstring"""







		A_						 =      42
		A_						 =      42
		A_						 =      42
		A_						 =      0.0
		A_						 =      False
		A_						 =      jnp.floataa
		A_						 =      False



		def 				_a				(			self    :				Any   ):






					"""simple docstring"""
					# self attention (or cross_attention if only_cross_attention is True)
					_snake_case              =      FlaxAttention(
					    self.dim  ,					self.n_heads  ,					self.d_head  ,					self.dropout  ,					self.use_memory_efficient_attention  ,					dtype=self.dtype   )
					# cross attention
					_snake_case              =      FlaxAttention(
					    self.dim  ,					self.n_heads  ,					self.d_head  ,					self.dropout  ,					self.use_memory_efficient_attention  ,					dtype=self.dtype   )
					_snake_case              =      FlaxFeedForward(dim=self.dim  ,					dropout=self.dropout  ,					dtype=self.dtype   )
					_snake_case              =      nn.LayerNorm(epsilon=1e-5  ,					dtype=self.dtype   )
					_snake_case              =      nn.LayerNorm(epsilon=1e-5  ,					dtype=self.dtype   )
					_snake_case              =      nn.LayerNorm(epsilon=1e-5  ,					dtype=self.dtype   )
					_snake_case              =      nn.Dropout(rate=self.dropout   )







		def __call__(			self    :				List[Any]  ,					__SCREAMING_SNAKE_CASE    :				Any  ,					__SCREAMING_SNAKE_CASE    :				List[str]  ,					__SCREAMING_SNAKE_CASE    :				Any=True   ):






					"""simple docstring"""
					# self attention
					_snake_case              =      hidden_states
					if self.only_cross_attention:
								_snake_case              =      self.attna(self.norma(__SCREAMING_SNAKE_CASE   )  ,					__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )
					else:
								_snake_case              =      self.attna(self.norma(__SCREAMING_SNAKE_CASE   )  ,					deterministic=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      hidden_states + residual

					# cross attention
					_snake_case              =      hidden_states
					_snake_case              =      self.attna(self.norma(__SCREAMING_SNAKE_CASE   )  ,					__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      hidden_states + residual

					# feed forward
					_snake_case              =      hidden_states
					_snake_case              =      self.ff(self.norma(__SCREAMING_SNAKE_CASE   )  ,					deterministic=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      hidden_states + residual

					return self.dropout_layer(__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )

class 	_SCREAMING_SNAKE_CASE							(     nn.Module							):






		"""simple docstring"""







		A_						 =      42
		A_						 =      42
		A_						 =      42
		A_						 =      1
		A_						 =      0.0
		A_						 =      False
		A_						 =      False
		A_						 =      jnp.floataa
		A_						 =      False



		def 				_a				(			self    :				Union[str, Any]   ):






					"""simple docstring"""
					_snake_case              =      nn.GroupNorm(num_groups=32  ,					epsilon=1e-5   )

					_snake_case              =      self.n_heads * self.d_head
					if self.use_linear_projection:
								_snake_case              =      nn.Dense(__SCREAMING_SNAKE_CASE  ,					dtype=self.dtype   )
					else:
								_snake_case              =      nn.Conv(
								    __SCREAMING_SNAKE_CASE  ,					kernel_size=(1, 1)  ,					strides=(1, 1)  ,					padding="VALID"  ,					dtype=self.dtype  ,					)

					_snake_case              =      [
					    FlaxBasicTransformerBlock(
					        __SCREAMING_SNAKE_CASE  ,					self.n_heads  ,					self.d_head  ,					dropout=self.dropout  ,					only_cross_attention=self.only_cross_attention  ,					dtype=self.dtype  ,					use_memory_efficient_attention=self.use_memory_efficient_attention  ,					)
					    for _ in range(self.depth   )
					]

					if self.use_linear_projection:
								_snake_case              =      nn.Dense(__SCREAMING_SNAKE_CASE  ,					dtype=self.dtype   )
					else:
								_snake_case              =      nn.Conv(
								    __SCREAMING_SNAKE_CASE  ,					kernel_size=(1, 1)  ,					strides=(1, 1)  ,					padding="VALID"  ,					dtype=self.dtype  ,					)

					_snake_case              =      nn.Dropout(rate=self.dropout   )







		def __call__(			self    :				Optional[int]  ,					__SCREAMING_SNAKE_CASE    :				List[str]  ,					__SCREAMING_SNAKE_CASE    :				Union[str, Any]  ,					__SCREAMING_SNAKE_CASE    :				Union[str, Any]=True   ):






					"""simple docstring"""
					_snake_case       , _snake_case       , _snake_case       , _snake_case              =      hidden_states.shape
					_snake_case              =      hidden_states
					_snake_case              =      self.norm(__SCREAMING_SNAKE_CASE   )
					if self.use_linear_projection:
								_snake_case              =      hidden_states.reshape(__SCREAMING_SNAKE_CASE  ,					height * width  ,					__SCREAMING_SNAKE_CASE   )
								_snake_case              =      self.proj_in(__SCREAMING_SNAKE_CASE   )
					else:
								_snake_case              =      self.proj_in(__SCREAMING_SNAKE_CASE   )
								_snake_case              =      hidden_states.reshape(__SCREAMING_SNAKE_CASE  ,					height * width  ,					__SCREAMING_SNAKE_CASE   )

					for transformer_block in self.transformer_blocks:
								_snake_case              =      transformer_block(__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )

					if self.use_linear_projection:
								_snake_case              =      self.proj_out(__SCREAMING_SNAKE_CASE   )
								_snake_case              =      hidden_states.reshape(__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )
					else:
								_snake_case              =      hidden_states.reshape(__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE  ,					__SCREAMING_SNAKE_CASE   )
								_snake_case              =      self.proj_out(__SCREAMING_SNAKE_CASE   )

					_snake_case              =      hidden_states + residual
					return self.dropout_layer(__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )

class 	_SCREAMING_SNAKE_CASE							(     nn.Module							):






		"""simple docstring"""







		A_						 =      42
		A_						 =      0.0
		A_						 =      jnp.floataa



		def 				_a				(			self    :				Optional[int]   ):






					"""simple docstring"""
					# The second linear layer needs to be called
					# net_2 for now to match the index of the Sequential layer
					_snake_case              =      FlaxGEGLU(self.dim  ,					self.dropout  ,					self.dtype   )
					_snake_case              =      nn.Dense(self.dim  ,					dtype=self.dtype   )







		def __call__(			self    :				Tuple  ,					__SCREAMING_SNAKE_CASE    :				List[str]  ,					__SCREAMING_SNAKE_CASE    :				List[Any]=True   ):






					"""simple docstring"""
					_snake_case              =      self.net_a(__SCREAMING_SNAKE_CASE  ,					deterministic=__SCREAMING_SNAKE_CASE   )
					_snake_case              =      self.net_a(__SCREAMING_SNAKE_CASE   )
					return hidden_states

class 	_SCREAMING_SNAKE_CASE							(     nn.Module							):






		"""simple docstring"""







		A_						 =      42
		A_						 =      0.0
		A_						 =      jnp.floataa



		def 				_a				(			self    :				Optional[Any]   ):






					"""simple docstring"""
					_snake_case              =      self.dim * 4
					_snake_case              =      nn.Dense(inner_dim * 2  ,					dtype=self.dtype   )
					_snake_case              =      nn.Dropout(rate=self.dropout   )







		def __call__(			self    :				Optional[int]  ,					__SCREAMING_SNAKE_CASE    :				Union[str, Any]  ,					__SCREAMING_SNAKE_CASE    :				List[str]=True   ):






					"""simple docstring"""
					_snake_case              =      self.proj(__SCREAMING_SNAKE_CASE   )
					_snake_case       , _snake_case              =      jnp.split(__SCREAMING_SNAKE_CASE  ,					2  ,					axis=2   )
					return self.dropout_layer(hidden_linear * nn.gelu(__SCREAMING_SNAKE_CASE   )  ,					deterministic=__SCREAMING_SNAKE_CASE   )

