






from manim import *




class        _lowerCAmelCase( UpperCAmelCase_      ):





       """simple docstring"""







       def     _a					(      self							):
         UpperCamelCase_:      int					   =       Rectangle(height=0.5						,    width=0.5							)
         UpperCamelCase_:      Optional[int]					   =       Rectangle(height=0.4_6						,    width=0.4_6							).set_stroke(width=0							)
         UpperCamelCase_:      Union[str, Any]					   =       Rectangle(height=0.2_5						,    width=0.2_5							)

         UpperCamelCase_:      List[Any]					   =       [mem.copy() for i in range(6							)]
         UpperCamelCase_:      Dict					   =       [mem.copy() for i in range(6							)]
         UpperCamelCase_:      List[Any]					   =       VGroup(*_lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0							)
         UpperCamelCase_:      Tuple					   =       VGroup(*_lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0							)
         UpperCamelCase_:      Dict					   =       VGroup(_lowerCamelCase						,    _lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0							)
         UpperCamelCase_:      List[str]					   =       Text('CPU'						,    font_size=2_4							)
         UpperCamelCase_:      List[Any]					   =       Group(_lowerCamelCase						,    _lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0.5						,    aligned_edge=_lowerCamelCase							)
         cpu.move_to([-2.5, -0.5, 0]							)
         self.add(_lowerCamelCase							)

         UpperCamelCase_:      Any					   =       [mem.copy() for i in range(4							)]
         UpperCamelCase_:      str					   =       VGroup(*_lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0							)
         UpperCamelCase_:      Dict					   =       Text('GPU'						,    font_size=2_4							)
         UpperCamelCase_:      Optional[int]					   =       Group(_lowerCamelCase						,    _lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0.5						,    aligned_edge=_lowerCamelCase							)
         gpu.move_to([-1, -1, 0]							)
         self.add(_lowerCamelCase							)

         UpperCamelCase_:      Any					   =       [mem.copy() for i in range(6							)]
         UpperCamelCase_:      Union[str, Any]					   =       VGroup(*_lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0							)

         UpperCamelCase_:      int					   =       Text('Model'						,    font_size=2_4							)
         UpperCamelCase_:      List[str]					   =       Group(_lowerCamelCase						,    _lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0.5						,    aligned_edge=_lowerCamelCase							)
         model.move_to([3, -1.0, 0]							)
         self.add(_lowerCamelCase							)

         UpperCamelCase_:      List[str]					   =       []
         UpperCamelCase_:      List[str]					   =       []

         for i, rect in enumerate(_lowerCamelCase							):
           rect.set_stroke(_lowerCamelCase							)

           UpperCamelCase_:      Union[str, Any]					   =       Rectangle(height=0.4_6 / 4						,    width=0.4_6 / 3							).set_stroke(width=0.0							).set_fill(_lowerCamelCase						,    opacity=0.7							)

           if i == 0:
             cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT							)						,    buff=0.0_2						,    direction=_lowerCamelCase							)
             cpu_target.set_x(cpu_target.get_x() + 0.1							)
           elif i == 3:
             cpu_target.next_to(model_cpu_arr[0]						,    direction=_lowerCamelCase						,    buff=0.0							)
           else:
             cpu_target.next_to(model_cpu_arr[i - 1]						,    direction=_lowerCamelCase						,    buff=0.0							)
           self.add(_lowerCamelCase							)
           model_cpu_arr.append(_lowerCamelCase							)

         self.add(*_lowerCamelCase						,    *_lowerCamelCase							)

         UpperCamelCase_:      int					   =       [meta_mem.copy() for i in range(6							)]
         UpperCamelCase_:      Dict					   =       [meta_mem.copy() for i in range(6							)]
         UpperCamelCase_:      Dict					   =       VGroup(*_lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0							)
         UpperCamelCase_:      Union[str, Any]					   =       VGroup(*_lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0							)
         UpperCamelCase_:      Union[str, Any]					   =       VGroup(_lowerCamelCase						,    _lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0							)
         UpperCamelCase_:      Any					   =       Text('Disk'						,    font_size=2_4							)
         UpperCamelCase_:      Dict					   =       Group(_lowerCamelCase						,    _lowerCamelCase							).arrange(_lowerCamelCase						,    buff=0.5						,    aligned_edge=_lowerCamelCase							)
         disk.move_to([-4.0, -1.2_5, 0]							)
         self.add(_lowerCamelCase						,    _lowerCamelCase							)

         UpperCamelCase_:      List[Any]					   =       []

         for i in range(6							):
           UpperCamelCase_:      List[str]					   =       fill.copy().set_fill(_lowerCamelCase						,    opacity=0.8							)
           target.move_to(disk_left_col_base[i]							).scale(0.5							)
           cpu_disk_arr.append(_lowerCamelCase							)

         self.add(*_lowerCamelCase							)

         UpperCamelCase_:      Any					   =       Square(side_length=2.2							)
         key.move_to([-5, 2, 0]							)

         UpperCamelCase_:      Any					   =       MarkupText(
             f'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''						,    font_size=1_8						,    )

         key_text.move_to([-5, 2.4, 0]							)

         self.add(_lowerCamelCase						,    _lowerCamelCase							)

         UpperCamelCase_:      int					   =       MarkupText(
             f'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''						,    font_size=1_8						,    )

         blue_text.next_to(_lowerCamelCase						,    DOWN * 2.4						,    aligned_edge=key_text.get_left()							)
         self.add(_lowerCamelCase							)

         UpperCamelCase_:      Union[str, Any]					   =       MarkupText(f'''The offloaded weights are all sent to the CPU.'''						,    font_size=2_4							)
         step_a.move_to([2, 2, 0]							)

         self.play(Write(_lowerCamelCase						,    run_time=3							)							)

         for i in range(6							):
           UpperCamelCase_:      str					   =       cpu_disk_arr[i]
           UpperCamelCase_:      Union[str, Any]					   =       rect.copy().set_fill(_lowerCamelCase						,    opacity=0.8							).scale(2.0							)
           cpa.generate_target()
           cpa.target.move_to(model_base[i]							)

           if i == 0:
             rect.set_fill(_lowerCamelCase						,    opacity=0.8							)
             rect.generate_target()
             rect.target.move_to(cpu_left_col_base[0]							).scale(2.0							)

             self.remove(
                 *_lowerCamelCase						,    *_lowerCamelCase						,    )

           else:
             rect.generate_target()
             rect.target.move_to(cpu_left_col_base[i]							).scale(2.0							)
           self.play(MoveToTarget(_lowerCamelCase							)						,    MoveToTarget(_lowerCamelCase							)						,    model_base[i].animate.set_stroke(_lowerCamelCase							)							)
         self.play(FadeOut(_lowerCamelCase							)							)

         UpperCamelCase_:      Any					   =       MarkupText(
             f'''Finally, hooks are added to each weight in the model\nto transfer the weights from CPU to GPU\n\t\tand back when needed.'''						,    font_size=2_4						,    )
         step_a.move_to([2, 2, 0]							)

         self.play(Write(_lowerCamelCase						,    run_time=3							)							)

         UpperCamelCase_:      Tuple					   =       []
         UpperCamelCase_:      str					   =       []
         for i in range(6							):
           UpperCamelCase_:      List[Any]					   =       Arrow(start=_lowerCamelCase						,    end=_lowerCamelCase						,    color=_lowerCamelCase						,    buff=0.5							)
           a.next_to(model_base[i].get_left()						,    _lowerCamelCase						,    buff=0.2							)
           arrows.append(_lowerCamelCase							)
           animations.append(Write(_lowerCamelCase							)							)
         self.play(*_lowerCamelCase							)
         self.wait()







import os
from shutil import copyfile
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_reformer import ReformerTokenizer
else:
   A_	:	Union[str, Any]				=				None


A_	:	List[Any]				=				logging.get_logger(__name__)


A_	:	List[str]				=				'▁'

A_	:	int				=				{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

A_	:	Optional[Any]				=				{
    'vocab_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'
        )
    },
    'tokenizer_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/tokenizer.json'
        )
    },
}

A_	:	List[Any]				=				{
    'google/reformer-crime-and-punishment': 524288,
}




class        _lowerCAmelCase( UpperCAmelCase_      ):





       """simple docstring"""







       a       :	int								=VOCAB_FILES_NAMES
       a       :	Optional[Any]								=PRETRAINED_VOCAB_FILES_MAP
       a       :	List[Any]								=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       a       :	Tuple								=['''input_ids''', '''attention_mask''']
       a       :	Optional[Any]								=ReformerTokenizer





       def __init__(      self						,    _lowerCamelCase=None						,    _lowerCamelCase=None						,    _lowerCamelCase="</s>"						,    _lowerCamelCase="<unk>"						,    _lowerCamelCase=[]						,    **_lowerCamelCase						,    ):
         super().__init__(
             _lowerCamelCase						,    tokenizer_file=_lowerCamelCase						,    eos_token=_lowerCamelCase						,    unk_token=_lowerCamelCase						,    additional_special_tokens=_lowerCamelCase						,    **_lowerCamelCase						,    )

         UpperCamelCase_:      Any					   =       vocab_file
         UpperCamelCase_:      str					   =       False if not self.vocab_file else True





       def     _a					(      self						,    _lowerCamelCase						,    _lowerCamelCase = None							):
         if not self.can_save_slow_tokenizer:
           raise ValueError(
               'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
               'tokenizer.'							)

         if not os.path.isdir(_lowerCamelCase							):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''							)
           return
         UpperCamelCase_:      str					   =       os.path.join(
             _lowerCamelCase						,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

         if os.path.abspath(self.vocab_file							) != os.path.abspath(_lowerCamelCase							):
           copyfile(self.vocab_file						,    _lowerCamelCase							)

         return (out_vocab_file,)