






'''simple docstring'''







import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase					   =       logging.get_logger(__name__)


__lowerCAmelCase					   =       {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all MVP models at https://huggingface.co/models?filter=mvp
__lowerCAmelCase					   =       {
    '''vocab_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json''',
    },
    '''added_tokens.json''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json''',
    },
    '''merges_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt''',
    },
}

__lowerCAmelCase					   =       {
    '''RUCAIBox/mvp''': 10_24,
}


@lru_cache()
def 			__UpperCamelCase				(			):
							"""simple docstring"""




							a_	    = (
							    list(range(ord('!'      )      ,					ord('~'      ) + 1      )      ) + list(range(ord('¡'      )      ,					ord('¬'      ) + 1      )      ) + list(range(ord('®'      )      ,					ord('ÿ'      ) + 1      )      )
							)
							a_	    = bs[:]
							a_	    = 0
							for b in range(2**8      ):
														if b not in bs:
																					bs.append(lowercase_      )
																					cs.append(2**8 + n      )
																					n += 1
							a_	    = [chr(lowercase_      ) for n in cs]
							return dict(zip(lowercase_      ,					lowercase_      )      )


def 			__UpperCamelCase				(			lowercase_					:     Union[str, Any]      ):
							"""simple docstring"""




							a_	    = set()
							a_	    = word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)      )
														a_	    = char
							return pairs




class      __SCREAMING_SNAKE_CASE					(lowercase__					):


		"""simple docstring"""


		_a     :   str			 =    VOCAB_FILES_NAMES
		_a     :   str			 =    PRETRAINED_VOCAB_FILES_MAP
		_a     :   List[Any]			 =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a     :   int			 =    ["""input_ids""", """attention_mask"""]







		def __init__( self      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__="replace"      ,  UpperCamelCase__="<s>"      ,  UpperCamelCase__="</s>"      ,  UpperCamelCase__="</s>"      ,  UpperCamelCase__="<s>"      ,  UpperCamelCase__="<unk>"      ,  UpperCamelCase__="<pad>"      ,  UpperCamelCase__="<mask>"      ,  UpperCamelCase__=False      ,  **UpperCamelCase__      ,  ):






									"""simple docstring"""


									a_	    = AddedToken(_A      ,  lstrip=_A      ,  rstrip=_A  ) if isinstance(_A      ,  _A  ) else bos_token
									a_	    = AddedToken(_A      ,  lstrip=_A      ,  rstrip=_A  ) if isinstance(_A      ,  _A  ) else eos_token
									a_	    = AddedToken(_A      ,  lstrip=_A      ,  rstrip=_A  ) if isinstance(_A      ,  _A  ) else sep_token
									a_	    = AddedToken(_A      ,  lstrip=_A      ,  rstrip=_A  ) if isinstance(_A      ,  _A  ) else cls_token
									a_	    = AddedToken(_A      ,  lstrip=_A      ,  rstrip=_A  ) if isinstance(_A      ,  _A  ) else unk_token
									a_	    = AddedToken(_A      ,  lstrip=_A      ,  rstrip=_A  ) if isinstance(_A      ,  _A  ) else pad_token

									# Mask token behave like a normal word, i.e. include the space before it
									a_	    = AddedToken(_A      ,  lstrip=_A      ,  rstrip=_A  ) if isinstance(_A      ,  _A  ) else mask_token

									super().__init__(
									    errors=_A      ,  bos_token=_A      ,  eos_token=_A      ,  unk_token=_A      ,  sep_token=_A      ,  cls_token=_A      ,  pad_token=_A      ,  mask_token=_A      ,  add_prefix_space=_A      ,  **_A      ,  )
									with open(_A      ,  encoding='utf-8'  ) as vocab_handle:
																a_	    = json.load(_A  )
									a_	    = {v: k for k, v in self.encoder.items()}
									a_	    = errors  # how to handle errors in decoding
									a_	    = bytes_to_unicode()
									a_	    = {v: k for k, v in self.byte_encoder.items()}
									with open(_A      ,  encoding='utf-8'  ) as merges_handle:
																a_	    = merges_handle.read().split('\n'  )[1:-1]
									a_	    = [tuple(merge.split()  ) for merge in bpe_merges]
									a_	    = dict(zip(_A      ,  range(len(_A  )  )  )  )
									a_	    = {}
									a_	    = add_prefix_space

									# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
									a_	    = re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'  )







		@property
		def       _a					( self  ):






									"""simple docstring"""


									return len(self.encoder  )







		def       _a					( self  ):






									"""simple docstring"""


									return dict(self.encoder      ,  **self.added_tokens_encoder  )







		def       _a					( self      ,  UpperCamelCase__  ):






									"""simple docstring"""


									if token in self.cache:
																return self.cache[token]
									a_	    = tuple(_A  )
									a_	    = get_pairs(_A  )

									if not pairs:
																return token

									while True:
																a_	    = min(_A      ,  key=lambda UpperCamelCase__  : self.bpe_ranks.get(_A      ,  float('inf'  )  )  )
																if bigram not in self.bpe_ranks:
																							break
																a_	    = bigram
																a_	    = []
																a_	    = 0
																while i < len(_A  ):
																							try:
																														a_	    = word.index(_A      ,  _A  )
																							except ValueError:
																														new_word.extend(word[i:]  )
																														break
																							else:
																														new_word.extend(word[i:j]  )
																														a_	    = j

																							if word[i] == first and i < len(_A  ) - 1 and word[i + 1] == second:
																														new_word.append(first + second  )
																														i += 2
																							else:
																														new_word.append(word[i]  )
																														i += 1
																a_	    = tuple(_A  )
																a_	    = new_word
																if len(_A  ) == 1:
																							break
																else:
																							a_	    = get_pairs(_A  )
									a_	    = ' '.join(_A  )
									a_	    = word
									return word







		def       _a					( self      ,  UpperCamelCase__  ):






									"""simple docstring"""


									a_	    = []
									for token in re.findall(self.pat      ,  _A  ):
																a_	    = ''.join(
																    self.byte_encoder[b] for b in token.encode('utf-8'  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_A  ).split(' '  )  )
									return bpe_tokens







		def       _a					( self      ,  UpperCamelCase__  ):






									"""simple docstring"""


									return self.encoder.get(_A      ,  self.encoder.get(self.unk_token  )  )







		def       _a					( self      ,  UpperCamelCase__  ):






									"""simple docstring"""


									return self.decoder.get(_A  )







		def       _a					( self      ,  UpperCamelCase__  ):






									"""simple docstring"""


									a_	    = ''.join(_A  )
									a_	    = bytearray([self.byte_decoder[c] for c in text]  ).decode('utf-8'      ,  errors=self.errors  )
									return text







		def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__ = None  ):






									"""simple docstring"""


									if not os.path.isdir(_A  ):
																logger.error(f'Vocabulary path ({save_directory}) should be a directory'  )
																return
									a_	    = os.path.join(
									    _A      ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
									a_	    = os.path.join(
									    _A      ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']  )

									with open(_A      ,  'w'      ,  encoding='utf-8'  ) as f:
																f.write(json.dumps(self.encoder      ,  indent=2      ,  sort_keys=_A      ,  ensure_ascii=_A  ) + '\n'  )

									a_	    = 0
									with open(_A      ,  'w'      ,  encoding='utf-8'  ) as writer:
																writer.write('#version: 0.2\n'  )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,  key=lambda UpperCamelCase__  : kv[1]  ):
																							if index != token_index:
																														logger.warning(
																														    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																														    ' Please check that the tokenizer is not corrupted!'  )
																														a_	    = token_index
																							writer.write(' '.join(_A  ) + '\n'  )
																							index += 1

									return vocab_file, merge_file







		def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__ = None  ):






									"""simple docstring"""


									if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									a_	    = [self.cls_token_id]
									a_	    = [self.sep_token_id]
									return cls + token_ids_a + sep + sep + token_ids_a + sep







		def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__ = None      ,  UpperCamelCase__ = False  ):






									"""simple docstring"""


									if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_A      ,  token_ids_a=_A      ,  already_has_special_tokens=_A  )

									if token_ids_a is None:
																return [1] + ([0] * len(_A  )) + [1]
									return [1] + ([0] * len(_A  )) + [1, 1] + ([0] * len(_A  )) + [1]







		def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__ = None  ):






									"""simple docstring"""


									a_	    = [self.sep_token_id]
									a_	    = [self.cls_token_id]

									if token_ids_a is None:
																return len(cls + token_ids_a + sep  ) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]







		def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__=False      ,  **UpperCamelCase__  ):






									"""simple docstring"""


									a_	    = kwargs.pop('add_prefix_space'      ,  self.add_prefix_space  )
									if (is_split_into_words or add_prefix_space) and (len(_A  ) > 0 and not text[0].isspace()):
																a_	    = ' ' + text
									return (text, kwargs)












'''simple docstring'''







import unittest

import numpy as np

from transformers import BlipTextConfig
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import is_torch_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask


if is_torch_available():
  import torch

  from transformers import BlipTextModel
  from transformers.models.blip.modeling_blip import BLIP_PRETRAINED_MODEL_ARCHIVE_LIST




class      __SCREAMING_SNAKE_CASE					:


  """simple docstring"""


  def __init__( self      ,  UpperCamelCase__      ,  UpperCamelCase__=12      ,  UpperCamelCase__=7      ,  UpperCamelCase__=True      ,  UpperCamelCase__=True      ,  UpperCamelCase__=True      ,  UpperCamelCase__=99      ,  UpperCamelCase__=32      ,  UpperCamelCase__=32      ,  UpperCamelCase__=5      ,  UpperCamelCase__=4      ,  UpperCamelCase__=37      ,  UpperCamelCase__=0.1      ,  UpperCamelCase__=0.1      ,  UpperCamelCase__=512      ,  UpperCamelCase__=0.02      ,  UpperCamelCase__=0      ,  UpperCamelCase__=None      ,  ):






         """simple docstring"""


         a_	    = parent
         a_	    = batch_size
         a_	    = seq_length
         a_	    = is_training
         a_	    = use_input_mask
         a_	    = use_labels
         a_	    = vocab_size
         a_	    = hidden_size
         a_	    = projection_dim
         a_	    = num_hidden_layers
         a_	    = num_attention_heads
         a_	    = intermediate_size
         a_	    = dropout
         a_	    = attention_dropout
         a_	    = max_position_embeddings
         a_	    = initializer_range
         a_	    = scope
         a_	    = bos_token_id







  def       _a					( self  ):






         """simple docstring"""


         a_	    = ids_tensor([self.batch_size, self.seq_length]      ,  self.vocab_size  )

         a_	    = None
         if self.use_input_mask:
                a_	    = random_attention_mask([self.batch_size, self.seq_length]  )

         if input_mask is not None:
                a_    ,  a_	    = input_mask.shape
                a_	    = np.random.randint(1      ,  seq_length - 1      ,  size=(batch_size,)  )
                for batch_idx, start_index in enumerate(UpperCamelCase__  ):
                       a_	    = 1
                       a_	    = 0

         a_	    = self.get_config()

         return config, input_ids, input_mask







  def       _a					( self  ):






         """simple docstring"""


         return BlipTextConfig(
             vocab_size=self.vocab_size      ,  hidden_size=self.hidden_size      ,  projection_dim=self.projection_dim      ,  num_hidden_layers=self.num_hidden_layers      ,  num_attention_heads=self.num_attention_heads      ,  intermediate_size=self.intermediate_size      ,  dropout=self.dropout      ,  attention_dropout=self.attention_dropout      ,  max_position_embeddings=self.max_position_embeddings      ,  initializer_range=self.initializer_range      ,  bos_token_id=self.bos_token_id      ,  )







  def       _a					( self      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = BlipTextModel(config=UpperCamelCase__  )
         model.to(UpperCamelCase__  )
         model.eval()
         with torch.no_grad():
                a_	    = model(UpperCamelCase__      ,  attention_mask=UpperCamelCase__  )
                a_	    = model(UpperCamelCase__  )
         self.parent.assertEqual(result.last_hidden_state.shape      ,  (self.batch_size, self.seq_length, self.hidden_size)  )
         self.parent.assertEqual(result.pooler_output.shape      ,  (self.batch_size, self.hidden_size)  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = self.prepare_config_and_inputs()
         a_    ,  a_    ,  a_	    = config_and_inputs
         a_	    = {'input_ids': input_ids, 'attention_mask': input_mask}
         return config, inputs_dict





@require_torch
class      __SCREAMING_SNAKE_CASE					(__A       ,					unittest.TestCase					):


  """simple docstring"""


  _a     :   Dict			 =    (BlipTextModel,) if is_torch_available() else ()
  _a     :   str			 =    False
  _a     :   str			 =    False
  _a     :   str			 =    False







  def       _a					( self  ):






         """simple docstring"""


         a_	    = BlipTextModelTester(self  )
         a_	    = ConfigTester(self      ,  config_class=UpperCamelCase__      ,  hidden_size=37  )







  def       _a					( self  ):






         """simple docstring"""


         self.config_tester.run_common_tests()







  def       _a					( self  ):






         """simple docstring"""


         a_	    = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*UpperCamelCase__  )







  def       _a					( self  ):






         """simple docstring"""


         pass







  def       _a					( self  ):






         """simple docstring"""


         pass







  @unittest.skip(reason='Blip does not use inputs_embeds'  )
  def       _a					( self  ):






         """simple docstring"""


         pass







  @unittest.skip(reason='BlipTextModel has no base class and is not available in MODEL_MAPPING'  )
  def       _a					( self  ):






         """simple docstring"""


         pass







  @unittest.skip(reason='BlipTextModel has no base class and is not available in MODEL_MAPPING'  )
  def       _a					( self  ):






         """simple docstring"""


         pass







  @slow
  def       _a					( self  ):






         """simple docstring"""


         for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                a_	    = BlipTextModel.from_pretrained(UpperCamelCase__  )
                self.assertIsNotNone(UpperCamelCase__  )







  def       _a					( self  ):






         """simple docstring"""


         super().test_pt_tf_model_equivalence(allow_missing_keys=UpperCamelCase__  )




