'''simple docstring'''






import colorsys
import io

import cva
import matplotlib as mpl
import matplotlib.colors as mplc
import matplotlib.figure as mplfigure
import numpy as np
import torch
from matplotlib.backends.backend_agg import FigureCanvasAgg

from utils import img_tensorize


_UpperCAmelCase						:    Tuple     					=					10_00







class lowercase_					:







   """simple docstring"""

   def __init__(			self			: Union[str, Any],   UpperCamelCase__			: Tuple,   UpperCamelCase__			: int=1.2,   UpperCamelCase__			: Tuple="g",   UpperCamelCase__			: Union[str, Any]=0.5,   UpperCamelCase__			: Optional[Any]="-",   UpperCamelCase__			: int="test_out.jpg",   UpperCamelCase__			: str=True,   UpperCamelCase__			: Union[str, Any]=False,   UpperCamelCase__			: Any=None,   UpperCamelCase__			: Union[str, Any]=None,   UpperCamelCase__			: Dict=0.7,   )     -> List[str]:
        if isinstance(__snake_case,   torch.Tensor    ):
             _A								=     img.numpy().astype('np.uint8'    )
        if isinstance(__snake_case,   __snake_case    ):
             _A								=     img_tensorize(__snake_case    )
        assert isinstance(__snake_case,   np.ndarray    )

        _A		,					_A								=     img.shape[1], img.shape[0]
        _A								=     mplfigure.Figure(frameon=__snake_case    )
        _A								=     fig.get_dpi()
        _A								=     (width * scale + 1e-2) / dpi
        _A								=     (height * scale + 1e-2) / dpi
        fig.set_size_inches(__snake_case,   __snake_case    )
        _A								=     fig.add_axes([0.0, 0.0, 1.0, 1.0]    )
        ax.axis('off'    )
        ax.set_xlim(0.0,   __snake_case    )
        ax.set_ylim(__snake_case    )

        _A								=     saveas
        _A								=     rgb
        _A								=     pynb
        _A								=     img
        _A								=     edgecolor
        _A								=     0.5
        _A								=     linestyle
        _A								=     int(np.sqrt(min(__snake_case,   __snake_case    )    ) * scale // 3    )
        _A								=     width
        _A								=     height
        _A								=     scale
        _A								=     fig
        _A								=     ax
        _A								=     pad
        _A								=     idaobj
        _A								=     idaattr
        _A								=     FigureCanvasAgg(__snake_case    )
   def   __UpperCAmelCase      (			self			: Union[str, Any],   UpperCamelCase__			: int,   UpperCamelCase__			: List[str]=None    )     -> Tuple:
        if color is None:
             _A								=     self.edgecolor
        ((_A)		,					(_A)		,					(_A)		,					(_A))								=     box
        _A								=     xa - xa
        _A								=     ya - ya
        self.ax.add_patch(
            mpl.patches.Rectangle(
                (xa, ya),   __snake_case,   __snake_case,   fill=__snake_case,   edgecolor=__snake_case,   linewidth=self.font_size // 3,   alpha=self.alpha,   linestyle=self.linestyle,   )    )
   def   __UpperCAmelCase      (			self			: List[str],   UpperCamelCase__			: Union[str, Any],   UpperCamelCase__			: Optional[int]=None,   UpperCamelCase__			: List[str]=None,   UpperCamelCase__			: Optional[Any]=None,   UpperCamelCase__			: List[str]=None    )     -> List[str]:
        if len(boxes.shape    ) > 2:
             _A								=     boxes[0]
        if len(obj_ids.shape    ) > 1:
             _A								=     obj_ids[0]
        if len(obj_scores.shape    ) > 1:
             _A								=     obj_scores[0]
        if len(attr_ids.shape    ) > 1:
             _A								=     attr_ids[0]
        if len(attr_scores.shape    ) > 1:
             _A								=     attr_scores[0]
        if isinstance(__snake_case,   torch.Tensor    ):
             _A								=     boxes.numpy()
        if isinstance(__snake_case,   __snake_case    ):
             _A								=     np.array(__snake_case    )
        assert isinstance(__snake_case,   np.ndarray    )
        _A								=     np.prod(boxes[:, 2:] - boxes[:, :2],   axis=1    )
        _A								=     np.argsort(-areas    ).tolist()
        _A								=     boxes[sorted_idxs] if boxes is not None else None
        _A								=     obj_ids[sorted_idxs] if obj_ids is not None else None
        _A								=     obj_scores[sorted_idxs] if obj_scores is not None else None
        _A								=     attr_ids[sorted_idxs] if attr_ids is not None else None
        _A								=     attr_scores[sorted_idxs] if attr_scores is not None else None

        _A								=     [self._random_color(maximum=1    ) for _ in range(len(__snake_case    )    )]
        _A								=     [assigned_colors[idx] for idx in sorted_idxs]
        if obj_ids is not None:
             _A								=     self._create_text_labels_attr(__snake_case,   __snake_case,   __snake_case,   __snake_case    )
             for i in range(len(__snake_case    )    ):
                  _A								=     assigned_colors[i]
                  self.add_box(boxes[i],   __snake_case    )
                  self.draw_labels(labels[i],   boxes[i],   __snake_case    )
   def   __UpperCAmelCase      (			self			: List[str],   UpperCamelCase__			: List[str],   UpperCamelCase__			: Optional[int],   UpperCamelCase__			: Tuple    )     -> Any:
        _A		,					_A		,					_A		,					_A								=     box
        _A								=     (xa, ya)
        _A								=     (ya - ya) * (xa - xa)
        _A								=     _SMALL_OBJ * self.scale
        if instance_area < small or ya - ya < 40 * self.scale:
             if ya >= self.height - 5:
                  _A								=     (xa, ya)
             else:
                  _A								=     (xa, ya)

        _A								=     (ya - ya) / np.sqrt(self.height * self.width    )
        _A								=     self._change_color_brightness(__snake_case,   brightness_factor=0.7    )
        _A								=     np.clip((height_ratio - 0.02) / 0.08 + 1,   1.2,   2    )
        font_size *= 0.75 * self.font_size

        self.draw_text(
            text=__snake_case,   position=__snake_case,   color=__snake_case,   )
   def   __UpperCAmelCase      (			self			: Union[str, Any],   UpperCamelCase__			: Optional[Any],   UpperCamelCase__			: int,   UpperCamelCase__			: Tuple="g",   UpperCamelCase__			: Optional[Any]="left",   )     -> List[str]:
        _A								=     0
        _A								=     self.font_size
        _A								=     np.maximum(list(mplc.to_rgb(__snake_case    )    ),   0.2    )
        _A								=     max(0.8,   np.max(__snake_case    )    )
        _A								=     {
            'facecolor': 'black',
            'alpha': self.alpha,
            'pad': self.pad,
            'edgecolor': 'none',
        }
        _A		,					_A								=     position
        self.ax.text(
            __snake_case,   __snake_case,   __snake_case,   size=font_size * self.scale,   family='sans-serif',   bbox=__snake_case,   verticalalignment='top',   horizontalalignment=__snake_case,   color=__snake_case,   zorder=10,   rotation=__snake_case,   )
   def   __UpperCAmelCase      (			self			: Optional[Any],   UpperCamelCase__			: Optional[int]=None    )     -> List[Any]:
        if saveas is None:
             _A								=     self.saveas
        if saveas.lower().endswith('.jpg'    ) or saveas.lower().endswith('.png'    ):
             cva.imwrite(
                 __snake_case,   self._get_buffer()[:, :, ::-1],   )
        else:
             self.fig.savefig(__snake_case    )
   def   __UpperCAmelCase      (			self			: Optional[int],   UpperCamelCase__			: Tuple,   UpperCamelCase__			: List[Any],   UpperCamelCase__			: int,   UpperCamelCase__			: Any    )     -> str:
        _A								=     [self.idaobj[i] for i in classes]
        _A								=     [self.idaattr[i] for i in attr_classes]
        _A								=     [
            f'{label} {score:.2f} {attr} {attr_score:.2f}'
            for label, score, attr, attr_score in zip(__snake_case,   __snake_case,   __snake_case,   __snake_case    )
        ]
        return labels
   def   __UpperCAmelCase      (			self			: List[Any],   UpperCamelCase__			: int,   UpperCamelCase__			: List[Any]    )     -> Dict:
        _A								=     [self.idaobj[i] for i in classes]
        if scores is not None:
             if labels is None:
                  _A								=     ['{:.0f}%'.format(s * 1_00    ) for s in scores]
             else:
                  _A								=     ['{} {:.0f}%'.format(__snake_case,   s * 1_00    ) for li, s in zip(__snake_case,   __snake_case    )]
        return labels
   def   __UpperCAmelCase      (			self			: List[str],   UpperCamelCase__			: Optional[Any]=2_55    )     -> List[str]:
        _A								=     np.random.randint(0,   len(_COLORS    )    )
        _A								=     _COLORS[idx] * maximum
        if not self.rgb:
             _A								=     ret[::-1]
        return ret
   def   __UpperCAmelCase      (			self			: List[Any]    )     -> List[Any]:
        if not self.pynb:
             _A		,					_A								=     self.canvas.print_to_buffer()
             if (width, height) != (self.width, self.height):
                  _A								=     cva.resize(self.img,   (width, height)    )
             else:
                  _A								=     self.img
        else:
             _A								=     io.BytesIO()  # works for cairo backend
             self.canvas.print_rgba(__snake_case    )
             _A		,					_A								=     self.width, self.height
             _A								=     buf.getvalue()
             _A								=     self.img

        _A								=     np.frombuffer(__snake_case,   dtype='uint8'    )
        _A								=     buffer.reshape(__snake_case,   __snake_case,   4    )
        _A		,					_A								=     np.split(__snake_case,   [3],   axis=2    )

        try:
             import numexpr as ne  # fuse them with numexpr

             _A								=     ne.evaluate('img * (1 - alpha / 255.0) + rgb * (alpha / 255.0)'    )
        except ImportError:
             _A								=     alpha.astype('float32'    ) / 255.0
             _A								=     img * (1 - alpha) + rgb * alpha

        return visualized_image.astype('uint8'    )

   def   __UpperCAmelCase      (			self			: Union[str, Any],   UpperCamelCase__			: Dict,   UpperCamelCase__			: int    )     -> Union[str, Any]:
        assert brightness_factor >= -1.0 and brightness_factor <= 1.0
        _A								=     mplc.to_rgb(__snake_case    )
        _A								=     colorsys.rgb_to_hls(*mplc.to_rgb(__snake_case    )    )
        _A								=     polygon_color[1] + (brightness_factor * polygon_color[1])
        _A								=     0.0 if modified_lightness < 0.0 else modified_lightness
        _A								=     1.0 if modified_lightness > 1.0 else modified_lightness
        _A								=     colorsys.hls_to_rgb(polygon_color[0],   __snake_case,   polygon_color[2]    )
        return modified_color






_UpperCAmelCase						:    Union[str, Any]     					=					(
    np.array(
        [
            0.0_00,
            0.4_47,
            0.7_41,
            0.8_50,
            0.3_25,
            0.0_98,
            0.9_29,
            0.6_94,
            0.1_25,
            0.4_94,
            0.1_84,
            0.5_56,
            0.4_66,
            0.6_74,
            0.1_88,
            0.3_01,
            0.7_45,
            0.9_33,
            0.6_35,
            0.0_78,
            0.1_84,
            0.3_00,
            0.3_00,
            0.3_00,
            0.6_00,
            0.6_00,
            0.6_00,
            1.0_00,
            0.0_00,
            0.0_00,
            1.0_00,
            0.5_00,
            0.0_00,
            0.7_49,
            0.7_49,
            0.0_00,
            0.0_00,
            1.0_00,
            0.0_00,
            0.0_00,
            0.0_00,
            1.0_00,
            0.6_67,
            0.0_00,
            1.0_00,
            0.3_33,
            0.3_33,
            0.0_00,
            0.3_33,
            0.6_67,
            0.0_00,
            0.3_33,
            1.0_00,
            0.0_00,
            0.6_67,
            0.3_33,
            0.0_00,
            0.6_67,
            0.6_67,
            0.0_00,
            0.6_67,
            1.0_00,
            0.0_00,
            1.0_00,
            0.3_33,
            0.0_00,
            1.0_00,
            0.6_67,
            0.0_00,
            1.0_00,
            1.0_00,
            0.0_00,
            0.0_00,
            0.3_33,
            0.5_00,
            0.0_00,
            0.6_67,
            0.5_00,
            0.0_00,
            1.0_00,
            0.5_00,
            0.3_33,
            0.0_00,
            0.5_00,
            0.3_33,
            0.3_33,
            0.5_00,
            0.3_33,
            0.6_67,
            0.5_00,
            0.3_33,
            1.0_00,
            0.5_00,
            0.6_67,
            0.0_00,
            0.5_00,
            0.6_67,
            0.3_33,
            0.5_00,
            0.6_67,
            0.6_67,
            0.5_00,
            0.6_67,
            1.0_00,
            0.5_00,
            1.0_00,
            0.0_00,
            0.5_00,
            1.0_00,
            0.3_33,
            0.5_00,
            1.0_00,
            0.6_67,
            0.5_00,
            1.0_00,
            1.0_00,
            0.5_00,
            0.0_00,
            0.3_33,
            1.0_00,
            0.0_00,
            0.6_67,
            1.0_00,
            0.0_00,
            1.0_00,
            1.0_00,
            0.3_33,
            0.0_00,
            1.0_00,
            0.3_33,
            0.3_33,
            1.0_00,
            0.3_33,
            0.6_67,
            1.0_00,
            0.3_33,
            1.0_00,
            1.0_00,
            0.6_67,
            0.0_00,
            1.0_00,
            0.6_67,
            0.3_33,
            1.0_00,
            0.6_67,
            0.6_67,
            1.0_00,
            0.6_67,
            1.0_00,
            1.0_00,
            1.0_00,
            0.0_00,
            1.0_00,
            1.0_00,
            0.3_33,
            1.0_00,
            1.0_00,
            0.6_67,
            1.0_00,
            0.3_33,
            0.0_00,
            0.0_00,
            0.5_00,
            0.0_00,
            0.0_00,
            0.6_67,
            0.0_00,
            0.0_00,
            0.8_33,
            0.0_00,
            0.0_00,
            1.0_00,
            0.0_00,
            0.0_00,
            0.0_00,
            0.1_67,
            0.0_00,
            0.0_00,
            0.3_33,
            0.0_00,
            0.0_00,
            0.5_00,
            0.0_00,
            0.0_00,
            0.6_67,
            0.0_00,
            0.0_00,
            0.8_33,
            0.0_00,
            0.0_00,
            1.0_00,
            0.0_00,
            0.0_00,
            0.0_00,
            0.1_67,
            0.0_00,
            0.0_00,
            0.3_33,
            0.0_00,
            0.0_00,
            0.5_00,
            0.0_00,
            0.0_00,
            0.6_67,
            0.0_00,
            0.0_00,
            0.8_33,
            0.0_00,
            0.0_00,
            1.0_00,
            0.0_00,
            0.0_00,
            0.0_00,
            0.1_43,
            0.1_43,
            0.1_43,
            0.8_57,
            0.8_57,
            0.8_57,
            1.0_00,
            1.0_00,
            1.0_00,
        ]
    )
    .astype(np.floataa)
    .reshape(-1, 3)
)




'''simple docstring'''






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_UpperCAmelCase						:    Optional[int]     					=					logging.get_logger(__name__)

_UpperCAmelCase						:    Optional[Any]     					=					{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_UpperCAmelCase						:    Optional[Any]     					=					{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

_UpperCAmelCase						:    int     					=					{
    '''openai-gpt''': 5_12,
}





def    _SCREAMING_SNAKE_CASE		(			__snake_case       : Any							):
     _A								=     text.strip()
     if not text:
          return []
     _A								=     text.split()
     return tokens







class lowercase_					(		_UpperCamelCase ):







   """simple docstring"""

   def __init__(			self			: Optional[int],   UpperCamelCase__			: Optional[int]=True,   UpperCamelCase__			: Optional[Any]=None,   UpperCamelCase__			: Optional[int]=True,   UpperCamelCase__			: Dict=None,   UpperCamelCase__			: Any=True,   )     -> str:
        if never_split is None:
             _A								=     []
        _A								=     do_lower_case
        _A								=     set(UpperCamelCase__    )
        _A								=     tokenize_chinese_chars
        _A								=     strip_accents
        _A								=     do_split_on_punc
   def   __UpperCAmelCase      (			self			: Any,   UpperCamelCase__			: List[str],   UpperCamelCase__			: int=None    )     -> Optional[Any]:
        _A								=     self.never_split.union(set(UpperCamelCase__    )    ) if never_split else self.never_split
        _A								=     self._clean_text(UpperCamelCase__    )

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        if self.tokenize_chinese_chars:
             _A								=     self._tokenize_chinese_chars(UpperCamelCase__    )
        # prevents treating the same character with different unicode codepoints as different characters
        _A								=     unicodedata.normalize('NFC',   UpperCamelCase__    )
        _A								=     whitespace_tokenize(UpperCamelCase__    )
        _A								=     []
        for token in orig_tokens:
             if token not in never_split:
                  if self.do_lower_case:
                       _A								=     token.lower()
                       if self.strip_accents is not False:
                            _A								=     self._run_strip_accents(UpperCamelCase__    )
                  elif self.strip_accents:
                       _A								=     self._run_strip_accents(UpperCamelCase__    )
             split_tokens.extend(self._run_split_on_punc(UpperCamelCase__,   UpperCamelCase__    )    )

        _A								=     whitespace_tokenize(' '.join(UpperCamelCase__    )    )
        return output_tokens
   def   __UpperCAmelCase      (			self			: List[Any],   UpperCamelCase__			: Dict    )     -> Dict:
        _A								=     unicodedata.normalize('NFD',   UpperCamelCase__    )
        _A								=     []
        for char in text:
             _A								=     unicodedata.category(UpperCamelCase__    )
             if cat == "Mn":
                  continue
             output.append(UpperCamelCase__    )
        return "".join(UpperCamelCase__    )
   def   __UpperCAmelCase      (			self			: Union[str, Any],   UpperCamelCase__			: Tuple,   UpperCamelCase__			: Any=None    )     -> Optional[Any]:
        if not self.do_split_on_punc or (never_split is not None and text in never_split):
             return [text]
        _A								=     list(UpperCamelCase__    )
        _A								=     0
        _A								=     True
        _A								=     []
        while i < len(UpperCamelCase__    ):
             _A								=     chars[i]
             if _is_punctuation(UpperCamelCase__    ):
                  output.append([char]    )
                  _A								=     True
             else:
                  if start_new_word:
                       output.append([]    )
                  _A								=     False
                  output[-1].append(UpperCamelCase__    )
             i += 1

        return ["".join(UpperCamelCase__    ) for x in output]
   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: str    )     -> int:
        _A								=     []
        for char in text:
             _A								=     ord(UpperCamelCase__    )
             if self._is_chinese_char(UpperCamelCase__    ):
                  output.append(' '    )
                  output.append(UpperCamelCase__    )
                  output.append(' '    )
             else:
                  output.append(UpperCamelCase__    )
        return "".join(UpperCamelCase__    )
   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: List[Any]    )     -> Union[str, Any]:
        # This defines a "chinese character" as anything in the CJK Unicode block:
        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
        #
        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
        # despite its name. The modern Korean Hangul alphabet is a different block,
        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
        # space-separated words, so they are not treated specially and handled
        # like the all of the other languages.
        if (
            (cp >= 0x4e_00 and cp <= 0x9f_ff)
            or (cp >= 0x34_00 and cp <= 0x4d_bf)  #
            or (cp >= 0x2_00_00 and cp <= 0x2_a6_df)  #
            or (cp >= 0x2_a7_00 and cp <= 0x2_b7_3f)  #
            or (cp >= 0x2_b7_40 and cp <= 0x2_b8_1f)  #
            or (cp >= 0x2_b8_20 and cp <= 0x2_ce_af)  #
            or (cp >= 0xf9_00 and cp <= 0xfa_ff)
            or (cp >= 0x2_f8_00 and cp <= 0x2_fa_1f)  #
        ):  #
             return True

        return False

   def   __UpperCAmelCase      (			self			: Optional[int],   UpperCamelCase__			: Union[str, Any]    )     -> str:
        _A								=     []
        for char in text:
             _A								=     ord(UpperCamelCase__    )
             if cp == 0 or cp == 0xff_fd or _is_control(UpperCamelCase__    ):
                  continue
             if _is_whitespace(UpperCamelCase__    ):
                  output.append(' '    )
             else:
                  output.append(UpperCamelCase__    )
        return "".join(UpperCamelCase__    )






def    _SCREAMING_SNAKE_CASE		(			__snake_case       : Optional[int]							):
     _A								=     set()
     _A								=     word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)							)
          _A								=     char
     return pairs





def    _SCREAMING_SNAKE_CASE		(			__snake_case       : Any							):
     _A								=     text.replace('—'      ,		'-'							)
     _A								=     text.replace('–'      ,		'-'							)
     _A								=     text.replace('―'      ,		'-'							)
     _A								=     text.replace('…'      ,		'...'							)
     _A								=     text.replace('´'      ,		'\''							)
     _A								=     re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'      ,		R' \1 '      ,		__snake_case							)
     _A								=     re.sub(R'\s*\n\s*'      ,		' \n '      ,		__snake_case							)
     _A								=     re.sub(R'[^\S\n]+'      ,		' '      ,		__snake_case							)
     return text.strip()







class lowercase_					(		_UpperCamelCase ):







   """simple docstring"""

   __lowerCAmelCase    						=      VOCAB_FILES_NAMES
   __lowerCAmelCase    						=      PRETRAINED_VOCAB_FILES_MAP
   __lowerCAmelCase    						=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __lowerCAmelCase    						=      ["input_ids", "attention_mask"]
   def __init__(			self			: Tuple,   UpperCamelCase__			: Tuple,   UpperCamelCase__			: List[str],   UpperCamelCase__			: Any="<unk>",   **UpperCamelCase__			: List[str]    )     -> Dict:
        super().__init__(unk_token=UpperCamelCase__,   **UpperCamelCase__    )

        try:
             import ftfy
             from spacy.lang.en import English

             _A								=     English()
             _A								=     _nlp.tokenizer
             _A								=     ftfy.fix_text
        except ImportError:
             logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'    )
             _A								=     BasicTokenizer(do_lower_case=UpperCamelCase__    )
             _A								=     None

        with open(UpperCamelCase__,   encoding='utf-8'    ) as vocab_handle:
             _A								=     json.load(UpperCamelCase__    )
        _A								=     {v: k for k, v in self.encoder.items()}
        with open(UpperCamelCase__,   encoding='utf-8'    ) as merges_handle:
             _A								=     merges_handle.read().split('\n'    )[1:-1]
        _A								=     [tuple(merge.split()    ) for merge in merges]
        _A								=     dict(zip(UpperCamelCase__,   range(len(UpperCamelCase__    )    )    )    )
        _A								=     {}
   @property
   def   __UpperCAmelCase      (			self			: Optional[int]    )     -> Tuple:
        return True
   @property
   def   __UpperCAmelCase      (			self			: str    )     -> List[str]:
        return len(self.encoder    )
   def   __UpperCAmelCase      (			self			: Tuple    )     -> Dict:
        return dict(self.encoder,   **self.added_tokens_encoder    )
   def   __UpperCAmelCase      (			self			: List[str],   UpperCamelCase__			: Dict    )     -> List[Any]:
        _A								=     tuple(token[:-1]    ) + (token[-1] + '</w>',)
        if token in self.cache:
             return self.cache[token]
        _A								=     get_pairs(UpperCamelCase__    )

        if not pairs:
             return token + "</w>"

        while True:
             _A								=     min(UpperCamelCase__,   key=lambda UpperCamelCase__    : self.bpe_ranks.get(UpperCamelCase__,   float('inf'    )    )    )
             if bigram not in self.bpe_ranks:
                  break
             _A		,					_A								=     bigram
             _A								=     []
             _A								=     0
             while i < len(UpperCamelCase__    ):
                  try:
                       _A								=     word.index(UpperCamelCase__,   UpperCamelCase__    )
                  except ValueError:
                       new_word.extend(word[i:]    )
                       break
                  else:
                       new_word.extend(word[i:j]    )
                       _A								=     j

                  if word[i] == first and i < len(UpperCamelCase__    ) - 1 and word[i + 1] == second:
                       new_word.append(first + second    )
                       i += 2
                  else:
                       new_word.append(word[i]    )
                       i += 1
             _A								=     tuple(UpperCamelCase__    )
             _A								=     new_word
             if len(UpperCamelCase__    ) == 1:
                  break
             else:
                  _A								=     get_pairs(UpperCamelCase__    )
        _A								=     ' '.join(UpperCamelCase__    )
        if word == "\n  </w>":
             _A								=     '\n</w>'
        _A								=     word
        return word
   def   __UpperCAmelCase      (			self			: Dict,   UpperCamelCase__			: Any    )     -> Union[str, Any]:
        _A								=     []
        if self.fix_text is None:
             # Using BERT's BasicTokenizer
             _A								=     self.nlp.tokenize(UpperCamelCase__    )
             for token in text:
                  split_tokens.extend(list(self.bpe(UpperCamelCase__    ).split(' '    )    )    )
        else:
             # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
             _A								=     self.nlp(text_standardize(self.fix_text(UpperCamelCase__    )    )    )
             for token in text:
                  split_tokens.extend(list(self.bpe(token.text.lower()    ).split(' '    )    )    )
        return split_tokens
   def   __UpperCAmelCase      (			self			: Any,   UpperCamelCase__			: List[Any]    )     -> List[Any]:
        return self.encoder.get(UpperCamelCase__,   self.encoder.get(self.unk_token    )    )
   def   __UpperCAmelCase      (			self			: int,   UpperCamelCase__			: List[str]    )     -> Tuple:
        return self.decoder.get(UpperCamelCase__,   self.unk_token    )
   def   __UpperCAmelCase      (			self			: Any,   UpperCamelCase__			: List[str]    )     -> Any:
        _A								=     ''.join(UpperCamelCase__    ).replace('</w>',   ' '    ).strip()
        return out_string

   def   __UpperCAmelCase      (			self			: Dict,   UpperCamelCase__			: str,   UpperCamelCase__			: Optional[str] = None    )     -> Tuple[str]:
        if not os.path.isdir(UpperCamelCase__    ):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory'    )
             return
        _A								=     os.path.join(
            UpperCamelCase__,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
        _A								=     os.path.join(
            UpperCamelCase__,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']    )

        with open(UpperCamelCase__,   'w',   encoding='utf-8'    ) as f:
             f.write(json.dumps(self.encoder,   indent=2,   sort_keys=UpperCamelCase__,   ensure_ascii=UpperCamelCase__    ) + '\n'    )

        _A								=     0
        with open(UpperCamelCase__,   'w',   encoding='utf-8'    ) as writer:
             writer.write('#version: 0.2\n'    )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),   key=lambda UpperCamelCase__    : kv[1]    ):
                  if index != token_index:
                       logger.warning(
                           f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                           ' Please check that the tokenizer is not corrupted!'    )
                       _A								=     token_index
                  writer.write(' '.join(UpperCamelCase__    ) + '\n'    )
                  index += 1

        return vocab_file, merge_file



