






from typing import Optional, Tuple

import flax.linen as nn
import jax
import jax.numpy as jnp
from configuration_hybrid_clip import HybridCLIPConfig
from flax.core.frozen_dict import FrozenDict

from transformers import FLAX_MODEL_MAPPING, FlaxCLIPVisionModel
from transformers.modeling_flax_utils import FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPOutput
from transformers.utils import logging


a    =       logging.get_logger(__name__)
class        __snake_case(					nn.Module      ):
 lowercase :       HybridCLIPConfig
 lowercase :       jnp.dtype		      =		jnp.floataa



 def 						UpperCAmelCase    (	self   :   int	):

    """simple docstring"""






    lowercase__						:	Optional[Any]         = self.config.text_config
    lowercase__						:	Tuple         = self.config.vision_config

    lowercase__						:	Any         = self.config.projection_dim
    lowercase__						:	Optional[int]         = text_config.hidden_size
    lowercase__						:	str         = vision_config.hidden_size

    lowercase__						:	Optional[Any]         = FLAX_MODEL_MAPPING[self.config.text_config.__class__].module_class
    lowercase__						:	Union[str, Any]         = FLAX_MODEL_MAPPING.get(self.config.vision_config.__class__ ,		UpperCAmelCase__	).module_class

    lowercase__						:	Optional[int]         = text_module(UpperCAmelCase__ ,		dtype=self.dtype	)
    lowercase__						:	str         = vision_module(UpperCAmelCase__ ,		dtype=self.dtype	)

    lowercase__						:	Optional[int]         = nn.Dense(
        self.projection_dim ,		dtype=self.dtype ,		kernel_init=jax.nn.initializers.normal(0.02	) ,		use_bias=UpperCAmelCase__ ,		)
    lowercase__						:	Tuple         = nn.Dense(
        self.projection_dim ,		dtype=self.dtype ,		kernel_init=jax.nn.initializers.normal(0.02	) ,		use_bias=UpperCAmelCase__ ,		)
    lowercase__						:	Optional[Any]         = self.param("""logit_scale""" ,		jax.nn.initializers.ones ,		[]	)







 def __call__(	self   :   Tuple ,		UpperCAmelCase__   :   List[Any]=None ,		UpperCAmelCase__   :   List[str]=None ,		UpperCAmelCase__   :   Optional[Any]=None ,		UpperCAmelCase__   :   Union[str, Any]=None ,		UpperCAmelCase__   :   List[str]=None ,		UpperCAmelCase__   :   bool = True ,		UpperCAmelCase__   :   str=None ,		UpperCAmelCase__   :   int=None ,		UpperCAmelCase__   :   Dict=None ,		):

    """simple docstring"""






    lowercase__						:	Optional[Any]         = return_dict if return_dict is not None else self.config.return_dict

    lowercase__						:	Union[str, Any]         = self.vision_model(
        pixel_values=UpperCAmelCase__ ,		deterministic=UpperCAmelCase__ ,		output_attentions=UpperCAmelCase__ ,		output_hidden_states=UpperCAmelCase__ ,		return_dict=UpperCAmelCase__ ,		)

    lowercase__						:	List[str]         = self.text_model(
        input_ids=UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		position_ids=UpperCAmelCase__ ,		deterministic=UpperCAmelCase__ ,		output_attentions=UpperCAmelCase__ ,		output_hidden_states=UpperCAmelCase__ ,		return_dict=UpperCAmelCase__ ,		)

    lowercase__						:	str         = vision_outputs[1]
    lowercase__						:	List[Any]         = self.visual_projection(UpperCAmelCase__	)

    lowercase__						:	Optional[Any]         = text_outputs[1]
    lowercase__						:	Tuple         = self.text_projection(UpperCAmelCase__	)

    # normalized features
    lowercase__						:	List[str]         = image_embeds / jnp.linalg.norm(UpperCAmelCase__ ,		axis=-1 ,		keepdims=UpperCAmelCase__	)
    lowercase__						:	List[Any]         = text_embeds / jnp.linalg.norm(UpperCAmelCase__ ,		axis=-1 ,		keepdims=UpperCAmelCase__	)

    # cosine similarity as logits
    lowercase__						:	List[Any]         = jnp.exp(self.logit_scale	)
    lowercase__						:	Dict         = jnp.matmul(UpperCAmelCase__ ,		image_embeds.T	) * logit_scale
    lowercase__						:	Any         = logits_per_text.T

    if not return_dict:
       return (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)

    return FlaxCLIPOutput(
        logits_per_image=UpperCAmelCase__ ,		logits_per_text=UpperCAmelCase__ ,		text_embeds=UpperCAmelCase__ ,		image_embeds=UpperCAmelCase__ ,		text_model_output=UpperCAmelCase__ ,		vision_model_output=UpperCAmelCase__ ,		)
class        __snake_case(					__UpperCamelCase      ):
 lowercase :       Optional[Any]		      =		HybridCLIPConfig
 lowercase :       List[Any]		      =		FlaxHybridCLIPModule



 def __init__(	self   :   Any ,		UpperCAmelCase__   :   HybridCLIPConfig ,		UpperCAmelCase__   :   Optional[Tuple] = None ,		UpperCAmelCase__   :   int = 0 ,		UpperCAmelCase__   :   jnp.dtype = jnp.floataa ,		**UpperCAmelCase__   :   Union[str, Any] ,		):

    """simple docstring"""






    if input_shape is None:
       lowercase__						:	Any         = ((1, 1), (1, config.vision_config.image_size, config.vision_config.image_size, 3))

    lowercase__						:	List[Any]         = self.module_class(config=UpperCAmelCase__ ,		dtype=UpperCAmelCase__ ,		**UpperCAmelCase__	)
    super().__init__(UpperCAmelCase__ ,		UpperCAmelCase__ ,		input_shape=UpperCAmelCase__ ,		seed=UpperCAmelCase__ ,		dtype=UpperCAmelCase__	)



 def 						UpperCAmelCase    (	self   :   List[Any] ,		UpperCAmelCase__   :   jax.random.PRNGKey ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   FrozenDict = None	):

    """simple docstring"""






    lowercase__						:	Dict         = jnp.zeros(input_shape[0] ,		dtype="""i4"""	)
    lowercase__						:	int         = jnp.broadcast_to(jnp.arange(jnp.atleast_ad(UpperCAmelCase__	).shape[-1]	) ,		input_shape[0]	)
    lowercase__						:	Optional[int]         = jnp.ones_like(UpperCAmelCase__	)
    lowercase__						:	Dict         = jnp.ones_like(UpperCAmelCase__	)

    lowercase__						:	List[str]         = jax.random.normal(UpperCAmelCase__ ,		input_shape[1]	)

    lowercase__     ,					lowercase__						:	Optional[int]         = jax.random.split(UpperCAmelCase__	)
    lowercase__						:	Optional[Any]         = {"""params""": params_rng, """dropout""": dropout_rng}

    return self.module.init(UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__	)["params"]



 def __call__(	self   :   str ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   List[str]=None ,		UpperCAmelCase__   :   List[str]=None ,		UpperCAmelCase__   :   List[str]=None ,		UpperCAmelCase__   :   dict = None ,		UpperCAmelCase__   :   jax.random.PRNGKey = None ,		UpperCAmelCase__   :   bool = False ,		UpperCAmelCase__   :   Optional[bool] = None ,		UpperCAmelCase__   :   Optional[bool] = None ,		UpperCAmelCase__   :   Optional[bool] = None ,		):

    """simple docstring"""






    lowercase__						:	Any         = output_attentions if output_attentions is not None else self.config.output_attentions
    lowercase__						:	List[str]         = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    lowercase__						:	Optional[Any]         = return_dict if return_dict is not None else self.config.return_dict

    if position_ids is None:
       lowercase__						:	List[str]         = jnp.broadcast_to(jnp.arange(jnp.atleast_ad(UpperCAmelCase__	).shape[-1]	) ,		input_ids.shape	)

    if token_type_ids is None:
       lowercase__						:	List[str]         = jnp.zeros_like(UpperCAmelCase__	)

    if attention_mask is None:
       lowercase__						:	Optional[int]         = jnp.ones_like(UpperCAmelCase__	)

    # Handle any PRNG if needed
    lowercase__						:	Optional[Any]         = {}
    if dropout_rng is not None:
       lowercase__						:	Optional[Any]         = dropout_rng

    return self.module.apply(
        {"""params""": params or self.params} ,		jnp.array(UpperCAmelCase__ ,		dtype="""i4"""	) ,		jnp.array(UpperCAmelCase__ ,		dtype=jnp.floataa	) ,		jnp.array(UpperCAmelCase__ ,		dtype="""i4"""	) ,		jnp.array(UpperCAmelCase__ ,		dtype="""i4"""	) ,		jnp.array(UpperCAmelCase__ ,		dtype="""i4"""	) ,		not train ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		rngs=UpperCAmelCase__ ,		)



 def 						UpperCAmelCase    (	self   :   Union[str, Any] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   str=None ,		UpperCAmelCase__   :   int=None ,		UpperCAmelCase__   :   Dict=None ,		UpperCAmelCase__   :   dict = None ,		UpperCAmelCase__   :   jax.random.PRNGKey = None ,		UpperCAmelCase__   :   List[str]=False ,		):

    """simple docstring"""






    if position_ids is None:
       lowercase__						:	str         = jnp.broadcast_to(jnp.arange(jnp.atleast_ad(UpperCAmelCase__	).shape[-1]	) ,		input_ids.shape	)

    if token_type_ids is None:
       lowercase__						:	int         = jnp.zeros_like(UpperCAmelCase__	)

    if attention_mask is None:
       lowercase__						:	Tuple         = jnp.ones_like(UpperCAmelCase__	)

    # Handle any PRNG if needed
    lowercase__						:	Union[str, Any]         = {}
    if dropout_rng is not None:
       lowercase__						:	Optional[Any]         = dropout_rng

    def _get_features(UpperCAmelCase__   :   Dict ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Optional[Any]	):
       lowercase__						:	str         = module.text_model(
           input_ids=UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		position_ids=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		deterministic=UpperCAmelCase__ ,		)
       lowercase__						:	Tuple         = text_outputs[1]
       lowercase__						:	Optional[int]         = module.text_projection(UpperCAmelCase__	)
       return text_features

    return self.module.apply(
        {"""params""": params or self.params} ,		jnp.array(UpperCAmelCase__ ,		dtype="""i4"""	) ,		jnp.array(UpperCAmelCase__ ,		dtype="""i4"""	) ,		jnp.array(UpperCAmelCase__ ,		dtype="""i4"""	) ,		jnp.array(UpperCAmelCase__ ,		dtype="""i4"""	) ,		not train ,		method=_get_features ,		rngs=UpperCAmelCase__ ,		)



 def 						UpperCAmelCase    (	self   :   Optional[Any] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   dict = None ,		UpperCAmelCase__   :   jax.random.PRNGKey = None ,		UpperCAmelCase__   :   List[Any]=False	):

    """simple docstring"""






    lowercase__						:	Tuple         = {}
    if dropout_rng is not None:
       lowercase__						:	Tuple         = dropout_rng

    def _get_features(UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   Union[str, Any]	):
       lowercase__						:	Optional[int]         = module.vision_model(pixel_values=UpperCAmelCase__ ,		deterministic=UpperCAmelCase__	)
       lowercase__						:	Dict         = vision_outputs[1]  # pooled_output
       lowercase__						:	List[str]         = module.visual_projection(UpperCAmelCase__	)
       return image_features

    return self.module.apply(
        {"""params""": params or self.params} ,		jnp.array(UpperCAmelCase__ ,		dtype=jnp.floataa	) ,		not train ,		method=_get_features ,		rngs=UpperCAmelCase__ ,		)







 @classmethod
 def 						UpperCAmelCase    (	cls   :   Tuple ,		UpperCAmelCase__   :   str = None ,		UpperCAmelCase__   :   str = None ,		*UpperCAmelCase__   :   List[str] ,		**UpperCAmelCase__   :   Dict ,		):

    """simple docstring"""






    lowercase__						:	List[str]         = {
        argument[len("""text_"""	) :]: value for argument, value in kwargs.items() if argument.startswith("""text_"""	)
    }

    lowercase__						:	List[str]         = {
        argument[len("""vision_"""	) :]: value for argument, value in kwargs.items() if argument.startswith("""vision_"""	)
    }

    # remove text, vision kwargs from kwargs
    for key in kwargs_text.keys():
       del kwargs["text_" + key]
    for key in kwargs_vision.keys():
       del kwargs["vision_" + key]

    # Load and initialize the text and vision model
    lowercase__						:	List[Any]         = kwargs_text.pop("""model""" ,		UpperCAmelCase__	)
    if text_model is None:
       assert (
           text_model_name_or_path is not None
       ), "If `model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
       from transformers import FlaxAutoModel

       if "config" not in kwargs_text:
          from transformers import AutoConfig

          lowercase__						:	Tuple         = AutoConfig.from_pretrained(UpperCAmelCase__	)
          lowercase__						:	Any         = text_config

       lowercase__						:	Optional[int]         = FlaxAutoModel.from_pretrained(UpperCAmelCase__ ,		*UpperCAmelCase__ ,		**UpperCAmelCase__	)

    lowercase__						:	Tuple         = kwargs_vision.pop("""model""" ,		UpperCAmelCase__	)
    if vision_model is None:
       assert (
           vision_model_name_or_path is not None
       ), "If `model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
       from transformers import FlaxAutoModel

       if "config" not in kwargs_vision:
          from transformers import AutoConfig

          lowercase__						:	str         = AutoConfig.from_pretrained(UpperCAmelCase__	)
          lowercase__						:	Tuple         = vision_config

       lowercase__						:	List[Any]         = FlaxAutoModel.from_pretrained(UpperCAmelCase__ ,		*UpperCAmelCase__ ,		**UpperCAmelCase__	)

    # instantiate config with corresponding kwargs
    lowercase__						:	List[str]         = kwargs.pop("""dtype""" ,		jnp.floataa	)
    lowercase__						:	Dict         = HybridCLIPConfig.from_text_vision_configs(text_model.config ,		vision_model.config ,		**UpperCAmelCase__	)

    # init model
    lowercase__						:	Optional[Any]         = cls(UpperCAmelCase__ ,		*UpperCAmelCase__ ,		dtype=UpperCAmelCase__ ,		**UpperCAmelCase__	)

    if vision_config.model_type == "clip":
       lowercase__						:	Optional[int]         = vision_model.params["""vision_model"""]
       lowercase__						:	Optional[int]         = vision_model.params["""visual_projection"""]["""kernel"""]
    else:
       lowercase__						:	Any         = vision_model.params

    lowercase__						:	List[Any]         = text_model.params

    return model










def       a__						(  A					):

   if not isinstance(A		,				A					) or number < 0:
      raise ValueError("""Input must be a non-negative integer"""					)

   lowercase__						:	str         = number & ~(number - 1)
   lowercase__						:	Optional[int]         = 0
   while intermediate:
      intermediate >>= 1
      index += 1
   return index - 1


if __name__ == "__main__":
 import doctest

 doctest.testmod(verbose=True)


