




'''simple docstring'''



import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation



def  lowercase__(		_UpperCamelCase  : torch.Tensor ,   _UpperCamelCase  : int ,   _UpperCamelCase  : bool = False ,   _UpperCamelCase  : float = 1 ,   _UpperCamelCase  : float = 1 ,   _UpperCamelCase  : int = 10000 ,   )->   Optional[Any]:






					"""simple docstring"""

					assert len(timesteps.shape				) == 1, "Timesteps should be a 1d-array"

					_UpperCamelCase											=   embedding_dim // 2
					_UpperCamelCase											=   -math.log(_UpperCamelCase				) * torch.arange(
					    start=0 ,   end=_UpperCamelCase ,   dtype=torch.floataa ,   device=timesteps.device				)
					_UpperCamelCase											=   exponent / (half_dim - downscale_freq_shift)

					_UpperCamelCase											=   torch.exp(_UpperCamelCase				)
					_UpperCamelCase											=   timesteps[:, None].float() * emb[None, :]

					# scale embeddings
					_UpperCamelCase											=   scale * emb

					# concat sine and cosine embeddings
					_UpperCamelCase											=   torch.cat([torch.sin(_UpperCamelCase				), torch.cos(_UpperCamelCase				)] ,   dim=-1				)

					# flip sine and cosine embeddings
					if flip_sin_to_cos:
										_UpperCamelCase											=   torch.cat([emb[:, half_dim:], emb[:, :half_dim]] ,   dim=-1				)

					# zero pad
					if embedding_dim % 2 == 1:
										_UpperCamelCase											=   torch.nn.functional.pad(_UpperCamelCase ,   (0, 1, 0, 0)				)
					return emb



def  lowercase__(		_UpperCamelCase  : List[Any] ,   _UpperCamelCase  : Dict ,   _UpperCamelCase  : Any=False ,   _UpperCamelCase  : Optional[Any]=0				)->   Tuple:






					"""simple docstring"""

					_UpperCamelCase											=   np.arange(_UpperCamelCase ,   dtype=np.floataa				)
					_UpperCamelCase											=   np.arange(_UpperCamelCase ,   dtype=np.floataa				)
					_UpperCamelCase											=   np.meshgrid(_UpperCamelCase ,   _UpperCamelCase				)  # here w goes first
					_UpperCamelCase											=   np.stack(_UpperCamelCase ,   axis=0				)

					_UpperCamelCase											=   grid.reshape([2, 1, grid_size, grid_size]				)
					_UpperCamelCase											=   get_ad_sincos_pos_embed_from_grid(_UpperCamelCase ,   _UpperCamelCase				)
					if cls_token and extra_tokens > 0:
										_UpperCamelCase											=   np.concatenate([np.zeros([extra_tokens, embed_dim]				), pos_embed] ,   axis=0				)
					return pos_embed



def  lowercase__(		_UpperCamelCase  : Union[str, Any] ,   _UpperCamelCase  : Any				)->   List[Any]:






					"""simple docstring"""

					if embed_dim % 2 != 0:
										raise ValueError("embed_dim must be divisible by 2"				)

					# use half of dimensions to encode grid_h
					_UpperCamelCase											=   get_ad_sincos_pos_embed_from_grid(embed_dim // 2 ,   grid[0]				)  # (H*W, D/2)
					_UpperCamelCase											=   get_ad_sincos_pos_embed_from_grid(embed_dim // 2 ,   grid[1]				)  # (H*W, D/2)

					_UpperCamelCase											=   np.concatenate([emb_h, emb_w] ,   axis=1				)  # (H*W, D)
					return emb



def  lowercase__(		_UpperCamelCase  : Tuple ,   _UpperCamelCase  : Any				)->   Dict:






					"""simple docstring"""

					if embed_dim % 2 != 0:
										raise ValueError("embed_dim must be divisible by 2"				)

					_UpperCamelCase											=   np.arange(embed_dim // 2 ,   dtype=np.floataa				)
					omega /= embed_dim / 2.0
					_UpperCamelCase											=   1.0 / 10000**omega  # (D/2,)

					_UpperCamelCase											=   pos.reshape(-1				)  # (M,)
					_UpperCamelCase											=   np.einsum("m,d->md" ,   _UpperCamelCase ,   _UpperCamelCase				)  # (M, D/2), outer product

					_UpperCamelCase											=   np.sin(_UpperCamelCase				)  # (M, D/2)
					_UpperCamelCase											=   np.cos(_UpperCamelCase				)  # (M, D/2)

					_UpperCamelCase											=   np.concatenate([emb_sin, emb_cos] ,   axis=1				)  # (M, D)
					return emb

class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_=2_24			,     A_=2_24			,     A_=16			,     A_=3			,     A_=7_68			,     A_=False			,     A_=True			,     A_=True			,     ):
								super().__init__()

								_UpperCamelCase											=   (height // patch_size) * (width // patch_size)
								_UpperCamelCase											=   flatten
								_UpperCamelCase											=   layer_norm

								_UpperCamelCase											=   nn.Convad(
								    A_			,     A_			,     kernel_size=(patch_size, patch_size)			,     stride=A_			,     bias=A_   )
								if layer_norm:
													_UpperCamelCase											=   nn.LayerNorm(A_			,     elementwise_affine=A_			,     eps=1e-6   )
								else:
													_UpperCamelCase											=   None

								_UpperCamelCase											=   get_ad_sincos_pos_embed(A_			,     int(num_patches**0.5   )   )
								self.register_buffer("pos_embed"			,     torch.from_numpy(A_   ).float().unsqueeze(0   )			,     persistent=A_   )




			def       a	(					self			,     A_   ):
								_UpperCamelCase											=   self.proj(A_   )
								if self.flatten:
													_UpperCamelCase											=   latent.flatten(2   ).transpose(1			,     2   )  # BCHW -> BNC
								if self.layer_norm:
													_UpperCamelCase											=   self.norm(A_   )
								return latent + self.pos_embed




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_			,     A_			,     A_ = "silu"			,     A_ = None			,     A_ = None			,     A_=None			,     ):
								super().__init__()

								_UpperCamelCase											=   nn.Linear(A_			,     A_   )

								if cond_proj_dim is not None:
													_UpperCamelCase											=   nn.Linear(A_			,     A_			,     bias=A_   )
								else:
													_UpperCamelCase											=   None

								_UpperCamelCase											=   get_activation(A_   )

								if out_dim is not None:
													_UpperCamelCase											=   out_dim
								else:
													_UpperCamelCase											=   time_embed_dim
								_UpperCamelCase											=   nn.Linear(A_			,     A_   )

								if post_act_fn is None:
													_UpperCamelCase											=   None
								else:
													_UpperCamelCase											=   get_activation(A_   )




			def       a	(					self			,     A_			,     A_=None   ):
								if condition is not None:
													_UpperCamelCase											=   sample + self.cond_proj(A_   )
								_UpperCamelCase											=   self.linear_a(A_   )

								if self.act is not None:
													_UpperCamelCase											=   self.act(A_   )

								_UpperCamelCase											=   self.linear_a(A_   )

								if self.post_act is not None:
													_UpperCamelCase											=   self.post_act(A_   )
								return sample




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_			,     A_			,     A_   ):
								super().__init__()
								_UpperCamelCase											=   num_channels
								_UpperCamelCase											=   flip_sin_to_cos
								_UpperCamelCase											=   downscale_freq_shift




			def       a	(					self			,     A_   ):
								_UpperCamelCase											=   get_timestep_embedding(
								    A_			,     self.num_channels			,     flip_sin_to_cos=self.flip_sin_to_cos			,     downscale_freq_shift=self.downscale_freq_shift			,     )
								return t_emb




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_ = 2_56			,     A_ = 1.0			,     A_=True			,     A_=True			,     A_=False   ):
								super().__init__()
								_UpperCamelCase											=   nn.Parameter(torch.randn(A_   ) * scale			,     requires_grad=A_   )
								_UpperCamelCase											=   log
								_UpperCamelCase											=   flip_sin_to_cos

								if set_W_to_weight:
													# to delete later
													_UpperCamelCase											=   nn.Parameter(torch.randn(A_   ) * scale			,     requires_grad=A_   )

													_UpperCamelCase											=   self.W




			def       a	(					self			,     A_   ):
								if self.log:
													_UpperCamelCase											=   torch.log(A_   )

								_UpperCamelCase											=   x[:, None] * self.weight[None, :] * 2 * np.pi

								if self.flip_sin_to_cos:
													_UpperCamelCase											=   torch.cat([torch.cos(A_   ), torch.sin(A_   )]			,     dim=-1   )
								else:
													_UpperCamelCase											=   torch.cat([torch.sin(A_   ), torch.cos(A_   )]			,     dim=-1   )
								return out




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_			,     A_			,     A_			,     A_			,     ):
								super().__init__()

								_UpperCamelCase											=   height
								_UpperCamelCase											=   width
								_UpperCamelCase											=   num_embed
								_UpperCamelCase											=   embed_dim

								_UpperCamelCase											=   nn.Embedding(self.num_embed			,     A_   )
								_UpperCamelCase											=   nn.Embedding(self.height			,     A_   )
								_UpperCamelCase											=   nn.Embedding(self.width			,     A_   )




			def       a	(					self			,     A_   ):
								_UpperCamelCase											=   self.emb(A_   )

								_UpperCamelCase											=   self.height_emb(torch.arange(self.height			,     device=index.device   ).view(1			,     self.height   )   )

								# 1 x H x D -> 1 x H x 1 x D
								_UpperCamelCase											=   height_emb.unsqueeze(2   )

								_UpperCamelCase											=   self.width_emb(torch.arange(self.width			,     device=index.device   ).view(1			,     self.width   )   )

								# 1 x W x D -> 1 x 1 x W x D
								_UpperCamelCase											=   width_emb.unsqueeze(1   )

								_UpperCamelCase											=   height_emb + width_emb

								# 1 x H x W x D -> 1 x L xD
								_UpperCamelCase											=   pos_emb.view(1			,     self.height * self.width			,     -1   )

								_UpperCamelCase											=   emb + pos_emb[:, : emb.shape[1], :]

								return emb




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_			,     A_			,     A_   ):
								super().__init__()
								_UpperCamelCase											=   dropout_prob > 0
								_UpperCamelCase											=   nn.Embedding(num_classes + use_cfg_embedding			,     A_   )
								_UpperCamelCase											=   num_classes
								_UpperCamelCase											=   dropout_prob




			def       a	(					self			,     A_			,     A_=None   ):
								if force_drop_ids is None:
													_UpperCamelCase											=   torch.rand(labels.shape[0]			,     device=labels.device   ) < self.dropout_prob
								else:
													_UpperCamelCase											=   torch.tensor(force_drop_ids == 1   )
								_UpperCamelCase											=   torch.where(A_			,     self.num_classes			,     A_   )
								return labels




			def       a	(					self			,     A_			,     A_=None   ):
								_UpperCamelCase											=   self.dropout_prob > 0
								if (self.training and use_dropout) or (force_drop_ids is not None):
													_UpperCamelCase											=   self.token_drop(A_			,     A_   )
								_UpperCamelCase											=   self.embedding_table(A_   )
								return embeddings




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_ = 10_24			,     A_ = 7_68			,     A_ = 7_68			,     A_ = 10			,     ):
								super().__init__()

								_UpperCamelCase											=   num_image_text_embeds
								_UpperCamelCase											=   nn.Linear(A_			,     self.num_image_text_embeds * cross_attention_dim   )
								_UpperCamelCase											=   nn.Linear(A_			,     A_   )




			def       a	(					self			,     A_			,     A_   ):
								_UpperCamelCase											=   text_embeds.shape[0]

								# image
								_UpperCamelCase											=   self.image_embeds(A_   )
								_UpperCamelCase											=   image_text_embeds.reshape(A_			,     self.num_image_text_embeds			,     -1   )

								# text
								_UpperCamelCase											=   self.text_proj(A_   )

								return torch.cat([image_text_embeds, text_embeds]			,     dim=1   )




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_ = 7_68			,     A_ = 7_68			,     A_ = 32			,     ):
								super().__init__()

								_UpperCamelCase											=   num_image_text_embeds
								_UpperCamelCase											=   nn.Linear(A_			,     self.num_image_text_embeds * cross_attention_dim   )
								_UpperCamelCase											=   nn.LayerNorm(A_   )




			def       a	(					self			,     A_   ):
								_UpperCamelCase											=   image_embeds.shape[0]

								# image
								_UpperCamelCase											=   self.image_embeds(A_   )
								_UpperCamelCase											=   image_embeds.reshape(A_			,     self.num_image_text_embeds			,     -1   )
								_UpperCamelCase											=   self.norm(A_   )
								return image_embeds




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_			,     A_			,     A_=0.1   ):
								super().__init__()

								_UpperCamelCase											=   Timesteps(num_channels=2_56			,     flip_sin_to_cos=A_			,     downscale_freq_shift=1   )
								_UpperCamelCase											=   TimestepEmbedding(in_channels=2_56			,     time_embed_dim=A_   )
								_UpperCamelCase											=   LabelEmbedding(A_			,     A_			,     A_   )




			def       a	(					self			,     A_			,     A_			,     A_=None   ):
								_UpperCamelCase											=   self.time_proj(A_   )
								_UpperCamelCase											=   self.timestep_embedder(timesteps_proj.to(dtype=A_   )   )  # (N, D)

								_UpperCamelCase											=   self.class_embedder(A_   )  # (N, D)

								_UpperCamelCase											=   timesteps_emb + class_labels  # (N, D)

								return conditioning




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_			,     A_			,     A_ = 64   ):
								super().__init__()
								_UpperCamelCase											=   nn.LayerNorm(A_   )
								_UpperCamelCase											=   AttentionPooling(A_			,     A_   )
								_UpperCamelCase											=   nn.Linear(A_			,     A_   )
								_UpperCamelCase											=   nn.LayerNorm(A_   )




			def       a	(					self			,     A_   ):
								_UpperCamelCase											=   self.norma(A_   )
								_UpperCamelCase											=   self.pool(A_   )
								_UpperCamelCase											=   self.proj(A_   )
								_UpperCamelCase											=   self.norma(A_   )
								return hidden_states




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_ = 7_68			,     A_ = 7_68			,     A_ = 15_36   ):
								super().__init__()
								_UpperCamelCase											=   nn.Linear(A_			,     A_   )
								_UpperCamelCase											=   nn.LayerNorm(A_   )
								_UpperCamelCase											=   nn.Linear(A_			,     A_   )




			def       a	(					self			,     A_			,     A_   ):
								# text
								_UpperCamelCase											=   self.text_proj(A_   )
								_UpperCamelCase											=   self.text_norm(A_   )

								# image
								_UpperCamelCase											=   self.image_proj(A_   )

								return time_image_embeds + time_text_embeds




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_ = 7_68			,     A_ = 15_36   ):
								super().__init__()
								_UpperCamelCase											=   nn.Linear(A_			,     A_   )
								_UpperCamelCase											=   nn.LayerNorm(A_   )




			def       a	(					self			,     A_   ):
								# image
								_UpperCamelCase											=   self.image_proj(A_   )
								_UpperCamelCase											=   self.image_norm(A_   )
								return time_image_embeds




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_ = 7_68			,     A_ = 15_36   ):
								super().__init__()
								_UpperCamelCase											=   nn.Linear(A_			,     A_   )
								_UpperCamelCase											=   nn.LayerNorm(A_   )
								_UpperCamelCase											=   nn.Sequential(
								    nn.Convad(3			,     16			,     3			,     padding=1   )			,     nn.SiLU()			,     nn.Convad(16			,     16			,     3			,     padding=1   )			,     nn.SiLU()			,     nn.Convad(16			,     32			,     3			,     padding=1			,     stride=2   )			,     nn.SiLU()			,     nn.Convad(32			,     32			,     3			,     padding=1   )			,     nn.SiLU()			,     nn.Convad(32			,     96			,     3			,     padding=1			,     stride=2   )			,     nn.SiLU()			,     nn.Convad(96			,     96			,     3			,     padding=1   )			,     nn.SiLU()			,     nn.Convad(96			,     2_56			,     3			,     padding=1			,     stride=2   )			,     nn.SiLU()			,     nn.Convad(2_56			,     4			,     3			,     padding=1   )			,     )




			def       a	(					self			,     A_			,     A_   ):
								# image
								_UpperCamelCase											=   self.image_proj(A_   )
								_UpperCamelCase											=   self.image_norm(A_   )
								_UpperCamelCase											=   self.input_hint_block(A_   )
								return time_image_embeds, hint




class     A_ (  nn.Module  ):

			'''simple docstring'''





			def __init__(					self			,     A_			,     A_			,     A_=None   ):
								super().__init__()
								_UpperCamelCase											=   dtype
								_UpperCamelCase											=   nn.Parameter(torch.randn(1			,     A_   ) / embed_dim**0.5   )
								_UpperCamelCase											=   nn.Linear(A_			,     A_			,     dtype=self.dtype   )
								_UpperCamelCase											=   nn.Linear(A_			,     A_			,     dtype=self.dtype   )
								_UpperCamelCase											=   nn.Linear(A_			,     A_			,     dtype=self.dtype   )
								_UpperCamelCase											=   num_heads
								_UpperCamelCase											=   embed_dim // self.num_heads




			def       a	(					self			,     A_   ):
								_UpperCamelCase				,	_UpperCamelCase				,	_UpperCamelCase											=   x.size()

								def shape(A_   ):
													# (bs, length, width) --> (bs, length, n_heads, dim_per_head)
													_UpperCamelCase											=   x.view(A_			,     -1			,     self.num_heads			,     self.dim_per_head   )
													# (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
													_UpperCamelCase											=   x.transpose(1			,     2   )
													# (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
													_UpperCamelCase											=   x.reshape(bs * self.num_heads			,     -1			,     self.dim_per_head   )
													# (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
													_UpperCamelCase											=   x.transpose(1			,     2   )
													return x

								_UpperCamelCase											=   x.mean(dim=1			,     keepdim=A_   ) + self.positional_embedding.to(x.dtype   )
								_UpperCamelCase											=   torch.cat([class_token, x]			,     dim=1   )  # (bs, length+1, width)

								# (bs*n_heads, class_token_length, dim_per_head)
								_UpperCamelCase											=   shape(self.q_proj(A_   )   )
								# (bs*n_heads, length+class_token_length, dim_per_head)
								_UpperCamelCase											=   shape(self.k_proj(A_   )   )
								_UpperCamelCase											=   shape(self.v_proj(A_   )   )

								# (bs*n_heads, class_token_length, length+class_token_length):
								_UpperCamelCase											=   1 / math.sqrt(math.sqrt(self.dim_per_head   )   )
								_UpperCamelCase											=   torch.einsum("bct,bcs->bts"			,     q * scale			,     k * scale   )  # More stable with f16 than dividing afterwards
								_UpperCamelCase											=   torch.softmax(weight.float()			,     dim=-1   ).type(weight.dtype   )

								# (bs*n_heads, dim_per_head, class_token_length)
								_UpperCamelCase											=   torch.einsum("bts,bcs->bct"			,     A_			,     A_   )

								# (bs, length+1, width)
								_UpperCamelCase											=   a.reshape(A_			,     -1			,     1   ).transpose(1			,     2   )

								return a[:, 0, :]  # cls_token










'''simple docstring'''



import collections
import inspect
import unittest

from transformers import DinatConfig
from transformers.testing_utils import require_natten, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch
			from torch import nn

			from transformers import DinatBackbone, DinatForImageClassification, DinatModel
			from transformers.models.dinat.modeling_dinat import DINAT_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
			from PIL import Image

			from transformers import AutoImageProcessor

class     A_ :

			'''simple docstring'''





			def __init__(					self			,     A_			,     A_=13			,     A_=64			,     A_=4			,     A_=3			,     A_=16			,     A_=[1, 2, 1]			,     A_=[2, 4, 8]			,     A_=3			,     A_=[[3], [1, 2], [1]]			,     A_=2.0			,     A_=True			,     A_=0.0			,     A_=0.0			,     A_=0.1			,     A_="gelu"			,     A_=True			,     A_=0.02			,     A_=1e-5			,     A_=True			,     A_=None			,     A_=True			,     A_=10			,     A_=["stage1", "stage2"]			,     A_=[1, 2]			,     ):
								_UpperCamelCase											=   parent
								_UpperCamelCase											=   batch_size
								_UpperCamelCase											=   image_size
								_UpperCamelCase											=   patch_size
								_UpperCamelCase											=   num_channels
								_UpperCamelCase											=   embed_dim
								_UpperCamelCase											=   depths
								_UpperCamelCase											=   num_heads
								_UpperCamelCase											=   kernel_size
								_UpperCamelCase											=   dilations
								_UpperCamelCase											=   mlp_ratio
								_UpperCamelCase											=   qkv_bias
								_UpperCamelCase											=   hidden_dropout_prob
								_UpperCamelCase											=   attention_probs_dropout_prob
								_UpperCamelCase											=   drop_path_rate
								_UpperCamelCase											=   hidden_act
								_UpperCamelCase											=   patch_norm
								_UpperCamelCase											=   layer_norm_eps
								_UpperCamelCase											=   initializer_range
								_UpperCamelCase											=   is_training
								_UpperCamelCase											=   scope
								_UpperCamelCase											=   use_labels
								_UpperCamelCase											=   num_labels
								_UpperCamelCase											=   out_features
								_UpperCamelCase											=   out_indices




			def       a	(					self   ):
								_UpperCamelCase											=   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]   )

								_UpperCamelCase											=   None
								if self.use_labels:
													_UpperCamelCase											=   ids_tensor([self.batch_size]			,     self.num_labels   )

								_UpperCamelCase											=   self.get_config()

								return config, pixel_values, labels




			def       a	(					self   ):
								return DinatConfig(
								    num_labels=self.num_labels			,     image_size=self.image_size			,     patch_size=self.patch_size			,     num_channels=self.num_channels			,     embed_dim=self.embed_dim			,     depths=self.depths			,     num_heads=self.num_heads			,     kernel_size=self.kernel_size			,     dilations=self.dilations			,     mlp_ratio=self.mlp_ratio			,     qkv_bias=self.qkv_bias			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     drop_path_rate=self.drop_path_rate			,     hidden_act=self.hidden_act			,     patch_norm=self.patch_norm			,     layer_norm_eps=self.layer_norm_eps			,     initializer_range=self.initializer_range			,     out_features=self.out_features			,     out_indices=self.out_indices			,     )




			def       a	(					self			,     A_			,     A_			,     A_   ):
								_UpperCamelCase											=   DinatModel(config=A_   )
								model.to(A_   )
								model.eval()
								_UpperCamelCase											=   model(A_   )

								_UpperCamelCase											=   _UpperCamelCase											=   (config.image_size // config.patch_size) // (2 ** (len(config.depths   ) - 1))
								_UpperCamelCase											=   int(config.embed_dim * 2 ** (len(config.depths   ) - 1)   )

								self.parent.assertEqual(
								    result.last_hidden_state.shape			,     (self.batch_size, expected_height, expected_width, expected_dim)   )




			def       a	(					self			,     A_			,     A_			,     A_   ):
								_UpperCamelCase											=   DinatForImageClassification(A_   )
								model.to(A_   )
								model.eval()
								_UpperCamelCase											=   model(A_			,     labels=A_   )
								self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.num_labels)   )

								# test greyscale images
								_UpperCamelCase											=   1
								_UpperCamelCase											=   DinatForImageClassification(A_   )
								model.to(A_   )
								model.eval()

								_UpperCamelCase											=   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]   )
								_UpperCamelCase											=   model(A_   )
								self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.num_labels)   )




			def       a	(					self			,     A_			,     A_			,     A_   ):
								_UpperCamelCase											=   DinatBackbone(config=A_   )
								model.to(A_   )
								model.eval()
								_UpperCamelCase											=   model(A_   )

								# verify hidden states
								self.parent.assertEqual(len(result.feature_maps   )			,     len(config.out_features   )   )
								self.parent.assertListEqual(list(result.feature_maps[0].shape   )			,     [self.batch_size, model.channels[0], 16, 16]   )

								# verify channels
								self.parent.assertEqual(len(model.channels   )			,     len(config.out_features   )   )

								# verify backbone works with out_features=None
								_UpperCamelCase											=   None
								_UpperCamelCase											=   DinatBackbone(config=A_   )
								model.to(A_   )
								model.eval()
								_UpperCamelCase											=   model(A_   )

								# verify feature maps
								self.parent.assertEqual(len(result.feature_maps   )			,     1   )
								self.parent.assertListEqual(list(result.feature_maps[0].shape   )			,     [self.batch_size, model.channels[-1], 4, 4]   )

								# verify channels
								self.parent.assertEqual(len(model.channels   )			,     1   )




			def       a	(					self   ):
								_UpperCamelCase											=   self.prepare_config_and_inputs()
								_UpperCamelCase				,	_UpperCamelCase				,	_UpperCamelCase											=   config_and_inputs
								_UpperCamelCase											=   {"pixel_values": pixel_values}
								return config, inputs_dict




@require_natten
@require_torch
class     A_ (  lowerCAmelCase_    ,			lowerCAmelCase_    ,			unittest.TestCase  ):

			'''simple docstring'''





			_lowerCAmelCase						=    (
			    (
			        DinatModel,
			        DinatForImageClassification,
			        DinatBackbone,
			    )
			    if is_torch_available()
			    else ()
			)
			_lowerCAmelCase						=    (
			    {"""feature-extraction""": DinatModel, """image-classification""": DinatForImageClassification}
			    if is_torch_available()
			    else {}
			)
			_lowerCAmelCase						=    False

			_lowerCAmelCase						=    False
			_lowerCAmelCase						=    False
			_lowerCAmelCase						=    False
			_lowerCAmelCase						=    False




			def       a	(					self   ):
								_UpperCamelCase											=   DinatModelTester(self   )
								_UpperCamelCase											=   ConfigTester(self			,     config_class=A_			,     embed_dim=37   )




			def       a	(					self   ):
								self.create_and_test_config_common_properties()
								self.config_tester.create_and_test_config_to_json_string()
								self.config_tester.create_and_test_config_to_json_file()
								self.config_tester.create_and_test_config_from_and_save_pretrained()
								self.config_tester.create_and_test_config_with_num_labels()
								self.config_tester.check_config_can_be_init_without_params()
								self.config_tester.check_config_arguments_init()




			def       a	(					self   ):
								return




			def       a	(					self   ):
								_UpperCamelCase											=   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*A_   )




			def       a	(					self   ):
								_UpperCamelCase											=   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_image_classification(*A_   )




			def       a	(					self   ):
								_UpperCamelCase											=   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_backbone(*A_   )




			@unittest.skip(reason="Dinat does not use inputs_embeds"   )
			def       a	(					self   ):
								pass




			@unittest.skip(reason="Dinat does not use feedforward chunking"   )
			def       a	(					self   ):
								pass




			def       a	(					self   ):
								_UpperCamelCase				,	_UpperCamelCase											=   self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
													_UpperCamelCase											=   model_class(A_   )
													self.assertIsInstance(model.get_input_embeddings()			,     (nn.Module)   )
													_UpperCamelCase											=   model.get_output_embeddings()
													self.assertTrue(x is None or isinstance(A_			,     nn.Linear   )   )




			def       a	(					self   ):
								_UpperCamelCase				,	_UpperCamelCase											=   self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
													_UpperCamelCase											=   model_class(A_   )
													_UpperCamelCase											=   inspect.signature(model.forward   )
													# signature.parameters is an OrderedDict => so arg_names order is deterministic
													_UpperCamelCase											=   [*signature.parameters.keys()]

													_UpperCamelCase											=   ["pixel_values"]
													self.assertListEqual(arg_names[:1]			,     A_   )




			def       a	(					self   ):
								self.skipTest("Dinat's attention operation is handled entirely by NATTEN."   )




			def       a	(					self			,     A_			,     A_			,     A_			,     A_   ):
								_UpperCamelCase											=   model_class(A_   )
								model.to(A_   )
								model.eval()

								with torch.no_grad():
													_UpperCamelCase											=   model(**self._prepare_for_class(A_			,     A_   )   )

								_UpperCamelCase											=   outputs.hidden_states

								_UpperCamelCase											=   getattr(
								    self.model_tester			,     "expected_num_hidden_layers"			,     len(self.model_tester.depths   ) + 1   )
								self.assertEqual(len(A_   )			,     A_   )

								# Dinat has a different seq_length
								_UpperCamelCase											=   (
								    config.patch_size
								    if isinstance(config.patch_size			,     collections.abc.Iterable   )
								    else (config.patch_size, config.patch_size)
								)

								_UpperCamelCase											=   image_size[0] // patch_size[0]
								_UpperCamelCase											=   image_size[1] // patch_size[1]

								self.assertListEqual(
								    list(hidden_states[0].shape[-3:]   )			,     [height, width, self.model_tester.embed_dim]			,     )

								if model_class.__name__ != "DinatBackbone":
													_UpperCamelCase											=   outputs.reshaped_hidden_states
													self.assertEqual(len(A_   )			,     A_   )

													_UpperCamelCase				,	_UpperCamelCase				,	_UpperCamelCase				,	_UpperCamelCase											=   reshaped_hidden_states[0].shape
													_UpperCamelCase											=   (
													    reshaped_hidden_states[0].view(A_			,     A_			,     A_			,     A_   ).permute(0			,     2			,     3			,     1   )
													)
													self.assertListEqual(
													    list(reshaped_hidden_states.shape[-3:]   )			,     [height, width, self.model_tester.embed_dim]			,     )




			def       a	(					self   ):
								_UpperCamelCase				,	_UpperCamelCase											=   self.model_tester.prepare_config_and_inputs_for_common()

								_UpperCamelCase											=   (
								    self.model_tester.image_size
								    if isinstance(self.model_tester.image_size			,     collections.abc.Iterable   )
								    else (self.model_tester.image_size, self.model_tester.image_size)
								)

								for model_class in self.all_model_classes:
													_UpperCamelCase											=   True
													self.check_hidden_states_output(A_			,     A_			,     A_			,     A_   )

													# check that output_hidden_states also work using config
													del inputs_dict["output_hidden_states"]
													_UpperCamelCase											=   True

													self.check_hidden_states_output(A_			,     A_			,     A_			,     A_   )




			@slow
			def       a	(					self   ):
								for model_name in DINAT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													_UpperCamelCase											=   DinatModel.from_pretrained(A_   )
													self.assertIsNotNone(A_   )




			def       a	(					self   ):
								_UpperCamelCase				,	_UpperCamelCase											=   self.model_tester.prepare_config_and_inputs_for_common()

								_UpperCamelCase											=   _config_zero_init(A_   )
								for model_class in self.all_model_classes:
													_UpperCamelCase											=   model_class(config=A_   )
													for name, param in model.named_parameters():
																		if "embeddings" not in name and param.requires_grad:
																							self.assertIn(
																							    ((param.data.mean() * 1e9).round() / 1e9).item()			,     [0.0, 1.0]			,     msg=F"Parameter {name} of model {model_class} seems not properly initialized"			,     )




@require_natten
@require_vision
@require_torch
class     A_ (  unittest.TestCase  ):

			'''simple docstring'''





			@cached_property
			def       a	(					self   ):
								return AutoImageProcessor.from_pretrained("shi-labs/dinat-mini-in1k-224"   ) if is_vision_available() else None




			@slow
			def       a	(					self   ):
								_UpperCamelCase											=   DinatForImageClassification.from_pretrained("shi-labs/dinat-mini-in1k-224"   ).to(A_   )
								_UpperCamelCase											=   self.default_image_processor

								_UpperCamelCase											=   Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"   )
								_UpperCamelCase											=   image_processor(images=A_			,     return_tensors="pt"   ).to(A_   )

								# forward pass
								with torch.no_grad():
													_UpperCamelCase											=   model(**A_   )

								# verify the logits
								_UpperCamelCase											=   torch.Size((1, 10_00)   )
								self.assertEqual(outputs.logits.shape			,     A_   )
								_UpperCamelCase											=   torch.tensor([-0.1545, -0.7667, 0.4642]   ).to(A_   )
								self.assertTrue(torch.allclose(outputs.logits[0, :3]			,     A_			,     atol=1e-4   )   )




@require_torch
@require_natten
class     A_ (  unittest.TestCase    ,			lowerCAmelCase_  ):

			'''simple docstring'''





			_lowerCAmelCase						=    (DinatBackbone,) if is_torch_available() else ()
			_lowerCAmelCase						=    DinatConfig




			def       a	(					self   ):
								_UpperCamelCase											=   DinatModelTester(self   )



