'''simple docstring'''






import inspect
import unittest

from transformers import SwinaSRConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import SwinaSRForImageSuperResolution, SwinaSRModel
 from transformers.models.swinasr.modeling_swinasr import SWIN2SR_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
 from PIL import Image

 from transformers import SwinaSRImageProcessor







class lowercase_					:







   """simple docstring"""

   def __init__(			self			: Tuple,   UpperCamelCase__			: List[str],   UpperCamelCase__			: Union[str, Any]=13,   UpperCamelCase__			: int=32,   UpperCamelCase__			: Optional[Any]=1,   UpperCamelCase__			: Dict=3,   UpperCamelCase__			: Any=16,   UpperCamelCase__			: Any=[1, 2, 1],   UpperCamelCase__			: Optional[Any]=[2, 2, 4],   UpperCamelCase__			: List[Any]=2,   UpperCamelCase__			: Tuple=2.0,   UpperCamelCase__			: List[str]=True,   UpperCamelCase__			: List[str]=0.0,   UpperCamelCase__			: str=0.0,   UpperCamelCase__			: Optional[int]=0.1,   UpperCamelCase__			: Union[str, Any]="gelu",   UpperCamelCase__			: Tuple=False,   UpperCamelCase__			: int=True,   UpperCamelCase__			: int=0.02,   UpperCamelCase__			: Tuple=1e-5,   UpperCamelCase__			: Optional[int]=True,   UpperCamelCase__			: Union[str, Any]=None,   UpperCamelCase__			: int=False,   UpperCamelCase__			: Union[str, Any]=2,   )     -> str:
        _A								=     parent
        _A								=     batch_size
        _A								=     image_size
        _A								=     patch_size
        _A								=     num_channels
        _A								=     embed_dim
        _A								=     depths
        _A								=     num_heads
        _A								=     window_size
        _A								=     mlp_ratio
        _A								=     qkv_bias
        _A								=     hidden_dropout_prob
        _A								=     attention_probs_dropout_prob
        _A								=     drop_path_rate
        _A								=     hidden_act
        _A								=     use_absolute_embeddings
        _A								=     patch_norm
        _A								=     layer_norm_eps
        _A								=     initializer_range
        _A								=     is_training
        _A								=     scope
        _A								=     use_labels
        _A								=     upscale

        # here we set some attributes to make tests pass
        _A								=     len(UpperCamelCase__    )
        _A								=     embed_dim
        _A								=     (image_size // patch_size) ** 2
   def   __UpperCAmelCase      (			self			: Optional[Any]    )     -> int:
        _A								=     floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]    )

        _A								=     None
        if self.use_labels:
             _A								=     ids_tensor([self.batch_size],   self.type_sequence_label_size    )

        _A								=     self.get_config()

        return config, pixel_values, labels
   def   __UpperCAmelCase      (			self			: Dict    )     -> Optional[int]:
        return SwinaSRConfig(
            image_size=self.image_size,   patch_size=self.patch_size,   num_channels=self.num_channels,   embed_dim=self.embed_dim,   depths=self.depths,   num_heads=self.num_heads,   window_size=self.window_size,   mlp_ratio=self.mlp_ratio,   qkv_bias=self.qkv_bias,   hidden_dropout_prob=self.hidden_dropout_prob,   attention_probs_dropout_prob=self.attention_probs_dropout_prob,   drop_path_rate=self.drop_path_rate,   hidden_act=self.hidden_act,   use_absolute_embeddings=self.use_absolute_embeddings,   path_norm=self.patch_norm,   layer_norm_eps=self.layer_norm_eps,   initializer_range=self.initializer_range,   upscale=self.upscale,   )
   def   __UpperCAmelCase      (			self			: List[str],   UpperCamelCase__			: List[str],   UpperCamelCase__			: Union[str, Any],   UpperCamelCase__			: str    )     -> List[str]:
        _A								=     SwinaSRModel(config=UpperCamelCase__    )
        model.to(UpperCamelCase__    )
        model.eval()
        _A								=     model(UpperCamelCase__    )

        self.parent.assertEqual(
            result.last_hidden_state.shape,   (self.batch_size, self.embed_dim, self.image_size, self.image_size)    )
   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: Tuple,   UpperCamelCase__			: Union[str, Any],   UpperCamelCase__			: Optional[int]    )     -> Dict:
        _A								=     SwinaSRForImageSuperResolution(UpperCamelCase__    )
        model.to(UpperCamelCase__    )
        model.eval()
        _A								=     model(UpperCamelCase__    )

        _A								=     self.image_size * self.upscale
        self.parent.assertEqual(
            result.reconstruction.shape,   (self.batch_size, self.num_channels, expected_image_size, expected_image_size)    )

   def   __UpperCAmelCase      (			self			: str    )     -> Union[str, Any]:
        _A								=     self.prepare_config_and_inputs()
        _A		,					_A		,					_A								=     config_and_inputs
        _A								=     {'pixel_values': pixel_values}
        return config, inputs_dict







@require_torch
class lowercase_					(		_UpperCamelCase						,						_UpperCamelCase						,						unittest.TestCase ):







   """simple docstring"""

   __lowerCAmelCase    						=      (SwinaSRModel, SwinaSRForImageSuperResolution) if is_torch_available() else ()
   __lowerCAmelCase    						=      {"feature-extraction": SwinaSRModel} if is_torch_available() else {}

   __lowerCAmelCase    						=      False
   __lowerCAmelCase    						=      False
   __lowerCAmelCase    						=      False
   __lowerCAmelCase    						=      False
   __lowerCAmelCase    						=      False
   def   __UpperCAmelCase      (			self			: Dict    )     -> int:
        _A								=     SwinaSRModelTester(self    )
        _A								=     ConfigTester(self,   config_class=UpperCamelCase__,   embed_dim=37    )
   def   __UpperCAmelCase      (			self			: int    )     -> Dict:
        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()
        self.config_tester.check_config_arguments_init()
   def   __UpperCAmelCase      (			self			: str    )     -> Any:
        _A								=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*UpperCamelCase__    )
   def   __UpperCAmelCase      (			self			: Any    )     -> Any:
        _A								=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_super_resolution(*UpperCamelCase__    )
   @unittest.skip(reason='Got `CUDA error: misaligned address` with PyTorch 2.0.0.'    )
   def   __UpperCAmelCase      (			self			: int    )     -> Any:
        pass
   @unittest.skip(reason='Swin2SR does not use inputs_embeds'    )
   def   __UpperCAmelCase      (			self			: Tuple    )     -> Optional[int]:
        pass
   @unittest.skip(reason='Swin2SR does not support training yet'    )
   def   __UpperCAmelCase      (			self			: Union[str, Any]    )     -> Union[str, Any]:
        pass
   @unittest.skip(reason='Swin2SR does not support training yet'    )
   def   __UpperCAmelCase      (			self			: List[str]    )     -> Dict:
        pass
   def   __UpperCAmelCase      (			self			: Optional[Any]    )     -> Any:
        _A		,					_A								=     self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
             _A								=     model_class(UpperCamelCase__    )
             self.assertIsInstance(model.get_input_embeddings(),   (nn.Module)    )
             _A								=     model.get_output_embeddings()
             self.assertTrue(x is None or isinstance(UpperCamelCase__,   nn.Linear    )    )
   def   __UpperCAmelCase      (			self			: str    )     -> Optional[Any]:
        _A		,					_A								=     self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
             _A								=     model_class(UpperCamelCase__    )
             _A								=     inspect.signature(model.forward    )
             # signature.parameters is an OrderedDict => so arg_names order is deterministic
             _A								=     [*signature.parameters.keys()]

             _A								=     ['pixel_values']
             self.assertListEqual(arg_names[:1],   UpperCamelCase__    )
   @slow
   def   __UpperCAmelCase      (			self			: Any    )     -> List[Any]:
        for model_name in SWIN2SR_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             _A								=     SwinaSRModel.from_pretrained(UpperCamelCase__    )
             self.assertIsNotNone(UpperCamelCase__    )
   def   __UpperCAmelCase      (			self			: Optional[int]    )     -> str:
        _A		,					_A								=     self.model_tester.prepare_config_and_inputs_for_common()

        _A								=     _config_zero_init(UpperCamelCase__    )
        for model_class in self.all_model_classes:
             _A								=     model_class(config=UpperCamelCase__    )
             for name, param in model.named_parameters():
                  if "logit_scale" in name:
                       continue
                  if param.requires_grad:
                       self.assertIn(
                           ((param.data.mean() * 1e9).round() / 1e9).item(),   [0.0, 1.0],   msg=f'Parameter {name} of model {model_class} seems not properly initialized',   )

   def   __UpperCAmelCase      (			self			: Union[str, Any]    )     -> Dict:
        _A		,					_A								=     self.model_tester.prepare_config_and_inputs_for_common()
        _A								=     True

        for model_class in self.all_model_classes:
             _A								=     True
             _A								=     False
             _A								=     True
             _A								=     model_class(UpperCamelCase__    )
             model.to(UpperCamelCase__    )
             model.eval()
             with torch.no_grad():
                  _A								=     model(**self._prepare_for_class(UpperCamelCase__,   UpperCamelCase__    )    )
             _A								=     outputs.attentions
             _A								=     len(self.model_tester.depths    )
             self.assertEqual(len(UpperCamelCase__    ),   UpperCamelCase__    )

             # check that output_attentions also work using config
             del inputs_dict["output_attentions"]
             _A								=     True
             _A								=     config.window_size**2
             _A								=     model_class(UpperCamelCase__    )
             model.to(UpperCamelCase__    )
             model.eval()
             with torch.no_grad():
                  _A								=     model(**self._prepare_for_class(UpperCamelCase__,   UpperCamelCase__    )    )
             _A								=     outputs.attentions
             self.assertEqual(len(UpperCamelCase__    ),   UpperCamelCase__    )

             self.assertListEqual(
                 list(attentions[0].shape[-3:]    ),   [self.model_tester.num_heads[0], window_size_squared, window_size_squared],   )
             _A								=     len(UpperCamelCase__    )

             # Check attention is always last and order is fine
             _A								=     True
             _A								=     True
             _A								=     model_class(UpperCamelCase__    )
             model.to(UpperCamelCase__    )
             model.eval()
             with torch.no_grad():
                  _A								=     model(**self._prepare_for_class(UpperCamelCase__,   UpperCamelCase__    )    )

             self.assertEqual(out_len + 1,   len(UpperCamelCase__    )    )

             _A								=     outputs.attentions

             self.assertEqual(len(UpperCamelCase__    ),   UpperCamelCase__    )

             self.assertListEqual(
                 list(self_attentions[0].shape[-3:]    ),   [self.model_tester.num_heads[0], window_size_squared, window_size_squared],   )







@require_vision
@require_torch
@slow
class lowercase_					(		unittest.TestCase ):







   """simple docstring"""

   def   __UpperCAmelCase      (			self			: Any    )     -> Optional[Any]:
        _A								=     SwinaSRImageProcessor()
        _A								=     SwinaSRForImageSuperResolution.from_pretrained('caidas/swin2SR-classical-sr-x2-64'    ).to(UpperCamelCase__    )

        _A								=     Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'    )
        _A								=     processor(images=UpperCamelCase__,   return_tensors='pt'    ).to(UpperCamelCase__    )

        # forward pass
        with torch.no_grad():
             _A								=     model(**UpperCamelCase__    )

        # verify the logits
        _A								=     torch.Size([1, 3, 9_76, 12_96]    )
        self.assertEqual(outputs.reconstruction.shape,   UpperCamelCase__    )
        _A								=     torch.tensor(
            [[0.5_458, 0.5_546, 0.5_638], [0.5_526, 0.5_565, 0.5_651], [0.5_396, 0.5_426, 0.5_621]]    ).to(UpperCamelCase__    )
        self.assertTrue(torch.allclose(outputs.reconstruction[0, 0, :3, :3],   UpperCamelCase__,   atol=1e-4    )    )




'''simple docstring'''






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_UpperCAmelCase						:    Optional[int]     					=					logging.get_logger(__name__)

_UpperCAmelCase						:    Optional[Any]     					=					{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_UpperCAmelCase						:    Optional[Any]     					=					{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

_UpperCAmelCase						:    int     					=					{
    '''openai-gpt''': 5_12,
}





def    _SCREAMING_SNAKE_CASE		(			__snake_case       : Any							):
     _A								=     text.strip()
     if not text:
          return []
     _A								=     text.split()
     return tokens







class lowercase_					(		_UpperCamelCase ):







   """simple docstring"""

   def __init__(			self			: Optional[int],   UpperCamelCase__			: Optional[int]=True,   UpperCamelCase__			: Optional[Any]=None,   UpperCamelCase__			: Optional[int]=True,   UpperCamelCase__			: Dict=None,   UpperCamelCase__			: Any=True,   )     -> str:
        if never_split is None:
             _A								=     []
        _A								=     do_lower_case
        _A								=     set(UpperCamelCase__    )
        _A								=     tokenize_chinese_chars
        _A								=     strip_accents
        _A								=     do_split_on_punc
   def   __UpperCAmelCase      (			self			: Any,   UpperCamelCase__			: List[str],   UpperCamelCase__			: int=None    )     -> Optional[Any]:
        _A								=     self.never_split.union(set(UpperCamelCase__    )    ) if never_split else self.never_split
        _A								=     self._clean_text(UpperCamelCase__    )

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        if self.tokenize_chinese_chars:
             _A								=     self._tokenize_chinese_chars(UpperCamelCase__    )
        # prevents treating the same character with different unicode codepoints as different characters
        _A								=     unicodedata.normalize('NFC',   UpperCamelCase__    )
        _A								=     whitespace_tokenize(UpperCamelCase__    )
        _A								=     []
        for token in orig_tokens:
             if token not in never_split:
                  if self.do_lower_case:
                       _A								=     token.lower()
                       if self.strip_accents is not False:
                            _A								=     self._run_strip_accents(UpperCamelCase__    )
                  elif self.strip_accents:
                       _A								=     self._run_strip_accents(UpperCamelCase__    )
             split_tokens.extend(self._run_split_on_punc(UpperCamelCase__,   UpperCamelCase__    )    )

        _A								=     whitespace_tokenize(' '.join(UpperCamelCase__    )    )
        return output_tokens
   def   __UpperCAmelCase      (			self			: List[Any],   UpperCamelCase__			: Dict    )     -> Dict:
        _A								=     unicodedata.normalize('NFD',   UpperCamelCase__    )
        _A								=     []
        for char in text:
             _A								=     unicodedata.category(UpperCamelCase__    )
             if cat == "Mn":
                  continue
             output.append(UpperCamelCase__    )
        return "".join(UpperCamelCase__    )
   def   __UpperCAmelCase      (			self			: Union[str, Any],   UpperCamelCase__			: Tuple,   UpperCamelCase__			: Any=None    )     -> Optional[Any]:
        if not self.do_split_on_punc or (never_split is not None and text in never_split):
             return [text]
        _A								=     list(UpperCamelCase__    )
        _A								=     0
        _A								=     True
        _A								=     []
        while i < len(UpperCamelCase__    ):
             _A								=     chars[i]
             if _is_punctuation(UpperCamelCase__    ):
                  output.append([char]    )
                  _A								=     True
             else:
                  if start_new_word:
                       output.append([]    )
                  _A								=     False
                  output[-1].append(UpperCamelCase__    )
             i += 1

        return ["".join(UpperCamelCase__    ) for x in output]
   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: str    )     -> int:
        _A								=     []
        for char in text:
             _A								=     ord(UpperCamelCase__    )
             if self._is_chinese_char(UpperCamelCase__    ):
                  output.append(' '    )
                  output.append(UpperCamelCase__    )
                  output.append(' '    )
             else:
                  output.append(UpperCamelCase__    )
        return "".join(UpperCamelCase__    )
   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: List[Any]    )     -> Union[str, Any]:
        # This defines a "chinese character" as anything in the CJK Unicode block:
        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
        #
        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
        # despite its name. The modern Korean Hangul alphabet is a different block,
        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
        # space-separated words, so they are not treated specially and handled
        # like the all of the other languages.
        if (
            (cp >= 0x4e_00 and cp <= 0x9f_ff)
            or (cp >= 0x34_00 and cp <= 0x4d_bf)  #
            or (cp >= 0x2_00_00 and cp <= 0x2_a6_df)  #
            or (cp >= 0x2_a7_00 and cp <= 0x2_b7_3f)  #
            or (cp >= 0x2_b7_40 and cp <= 0x2_b8_1f)  #
            or (cp >= 0x2_b8_20 and cp <= 0x2_ce_af)  #
            or (cp >= 0xf9_00 and cp <= 0xfa_ff)
            or (cp >= 0x2_f8_00 and cp <= 0x2_fa_1f)  #
        ):  #
             return True

        return False

   def   __UpperCAmelCase      (			self			: Optional[int],   UpperCamelCase__			: Union[str, Any]    )     -> str:
        _A								=     []
        for char in text:
             _A								=     ord(UpperCamelCase__    )
             if cp == 0 or cp == 0xff_fd or _is_control(UpperCamelCase__    ):
                  continue
             if _is_whitespace(UpperCamelCase__    ):
                  output.append(' '    )
             else:
                  output.append(UpperCamelCase__    )
        return "".join(UpperCamelCase__    )






def    _SCREAMING_SNAKE_CASE		(			__snake_case       : Optional[int]							):
     _A								=     set()
     _A								=     word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)							)
          _A								=     char
     return pairs





def    _SCREAMING_SNAKE_CASE		(			__snake_case       : Any							):
     _A								=     text.replace('—'      ,		'-'							)
     _A								=     text.replace('–'      ,		'-'							)
     _A								=     text.replace('―'      ,		'-'							)
     _A								=     text.replace('…'      ,		'...'							)
     _A								=     text.replace('´'      ,		'\''							)
     _A								=     re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'      ,		R' \1 '      ,		__snake_case							)
     _A								=     re.sub(R'\s*\n\s*'      ,		' \n '      ,		__snake_case							)
     _A								=     re.sub(R'[^\S\n]+'      ,		' '      ,		__snake_case							)
     return text.strip()







class lowercase_					(		_UpperCamelCase ):







   """simple docstring"""

   __lowerCAmelCase    						=      VOCAB_FILES_NAMES
   __lowerCAmelCase    						=      PRETRAINED_VOCAB_FILES_MAP
   __lowerCAmelCase    						=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __lowerCAmelCase    						=      ["input_ids", "attention_mask"]
   def __init__(			self			: Tuple,   UpperCamelCase__			: Tuple,   UpperCamelCase__			: List[str],   UpperCamelCase__			: Any="<unk>",   **UpperCamelCase__			: List[str]    )     -> Dict:
        super().__init__(unk_token=UpperCamelCase__,   **UpperCamelCase__    )

        try:
             import ftfy
             from spacy.lang.en import English

             _A								=     English()
             _A								=     _nlp.tokenizer
             _A								=     ftfy.fix_text
        except ImportError:
             logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'    )
             _A								=     BasicTokenizer(do_lower_case=UpperCamelCase__    )
             _A								=     None

        with open(UpperCamelCase__,   encoding='utf-8'    ) as vocab_handle:
             _A								=     json.load(UpperCamelCase__    )
        _A								=     {v: k for k, v in self.encoder.items()}
        with open(UpperCamelCase__,   encoding='utf-8'    ) as merges_handle:
             _A								=     merges_handle.read().split('\n'    )[1:-1]
        _A								=     [tuple(merge.split()    ) for merge in merges]
        _A								=     dict(zip(UpperCamelCase__,   range(len(UpperCamelCase__    )    )    )    )
        _A								=     {}
   @property
   def   __UpperCAmelCase      (			self			: Optional[int]    )     -> Tuple:
        return True
   @property
   def   __UpperCAmelCase      (			self			: str    )     -> List[str]:
        return len(self.encoder    )
   def   __UpperCAmelCase      (			self			: Tuple    )     -> Dict:
        return dict(self.encoder,   **self.added_tokens_encoder    )
   def   __UpperCAmelCase      (			self			: List[str],   UpperCamelCase__			: Dict    )     -> List[Any]:
        _A								=     tuple(token[:-1]    ) + (token[-1] + '</w>',)
        if token in self.cache:
             return self.cache[token]
        _A								=     get_pairs(UpperCamelCase__    )

        if not pairs:
             return token + "</w>"

        while True:
             _A								=     min(UpperCamelCase__,   key=lambda UpperCamelCase__    : self.bpe_ranks.get(UpperCamelCase__,   float('inf'    )    )    )
             if bigram not in self.bpe_ranks:
                  break
             _A		,					_A								=     bigram
             _A								=     []
             _A								=     0
             while i < len(UpperCamelCase__    ):
                  try:
                       _A								=     word.index(UpperCamelCase__,   UpperCamelCase__    )
                  except ValueError:
                       new_word.extend(word[i:]    )
                       break
                  else:
                       new_word.extend(word[i:j]    )
                       _A								=     j

                  if word[i] == first and i < len(UpperCamelCase__    ) - 1 and word[i + 1] == second:
                       new_word.append(first + second    )
                       i += 2
                  else:
                       new_word.append(word[i]    )
                       i += 1
             _A								=     tuple(UpperCamelCase__    )
             _A								=     new_word
             if len(UpperCamelCase__    ) == 1:
                  break
             else:
                  _A								=     get_pairs(UpperCamelCase__    )
        _A								=     ' '.join(UpperCamelCase__    )
        if word == "\n  </w>":
             _A								=     '\n</w>'
        _A								=     word
        return word
   def   __UpperCAmelCase      (			self			: Dict,   UpperCamelCase__			: Any    )     -> Union[str, Any]:
        _A								=     []
        if self.fix_text is None:
             # Using BERT's BasicTokenizer
             _A								=     self.nlp.tokenize(UpperCamelCase__    )
             for token in text:
                  split_tokens.extend(list(self.bpe(UpperCamelCase__    ).split(' '    )    )    )
        else:
             # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
             _A								=     self.nlp(text_standardize(self.fix_text(UpperCamelCase__    )    )    )
             for token in text:
                  split_tokens.extend(list(self.bpe(token.text.lower()    ).split(' '    )    )    )
        return split_tokens
   def   __UpperCAmelCase      (			self			: Any,   UpperCamelCase__			: List[Any]    )     -> List[Any]:
        return self.encoder.get(UpperCamelCase__,   self.encoder.get(self.unk_token    )    )
   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: List[str]    )     -> Tuple:
        return self.decoder.get(UpperCamelCase__,   self.unk_token    )
   def   __UpperCAmelCase      (			self			: Any,   UpperCamelCase__			: List[str]    )     -> Any:
        _A								=     ''.join(UpperCamelCase__    ).replace('</w>',   ' '    ).strip()
        return out_string

   def   __UpperCAmelCase      (			self			: Dict,   UpperCamelCase__			: str,   UpperCamelCase__			: Optional[str] = None    )     -> Tuple[str]:
        if not os.path.isdir(UpperCamelCase__    ):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory'    )
             return
        _A								=     os.path.join(
            UpperCamelCase__,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
        _A								=     os.path.join(
            UpperCamelCase__,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']    )

        with open(UpperCamelCase__,   'w',   encoding='utf-8'    ) as f:
             f.write(json.dumps(self.encoder,   indent=2,   sort_keys=UpperCamelCase__,   ensure_ascii=UpperCamelCase__    ) + '\n'    )

        _A								=     0
        with open(UpperCamelCase__,   'w',   encoding='utf-8'    ) as writer:
             writer.write('#version: 0.2\n'    )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),   key=lambda UpperCamelCase__    : kv[1]    ):
                  if index != token_index:
                       logger.warning(
                           f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                           ' Please check that the tokenizer is not corrupted!'    )
                       _A								=     token_index
                  writer.write(' '.join(UpperCamelCase__    ) + '\n'    )
                  index += 1

        return vocab_file, merge_file



