




'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import AddedToken, PreTrainedTokenizerFast
from ...utils import logging


UpperCamelCase_		:    Any									=		logging.get_logger(__name__)

UpperCamelCase_		:    Union[str, Any]									=		{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_		:    Optional[Any]									=		{
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/tokenizer.json''',
    },
}

class      _a       (		__lowerCamelCase							):






     def __init__(				self      ,_SCREAMING_SNAKE_CASE=None      ,_SCREAMING_SNAKE_CASE=None      ,_SCREAMING_SNAKE_CASE=False      ,_SCREAMING_SNAKE_CASE=True      ,_SCREAMING_SNAKE_CASE=False      ,_SCREAMING_SNAKE_CASE="<s>"      ,_SCREAMING_SNAKE_CASE="</s>"      ,_SCREAMING_SNAKE_CASE="<unk>"      ,_SCREAMING_SNAKE_CASE="<sep>"      ,_SCREAMING_SNAKE_CASE="<pad>"      ,_SCREAMING_SNAKE_CASE="<cls>"      ,_SCREAMING_SNAKE_CASE="<mask>"      ,_SCREAMING_SNAKE_CASE=["<eop>", "<eod>"]      ,**_SCREAMING_SNAKE_CASE      ,)							->    Optional[Any]:
          _snake_case								=   AddedToken(_SCREAMING_SNAKE_CASE      ,lstrip=_SCREAMING_SNAKE_CASE      ,rstrip=_SCREAMING_SNAKE_CASE					) if isinstance(_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE					) else mask_token

          super().__init__(
              vocab_file=_SCREAMING_SNAKE_CASE      ,tokenizer_file=_SCREAMING_SNAKE_CASE      ,do_lower_case=_SCREAMING_SNAKE_CASE      ,remove_space=_SCREAMING_SNAKE_CASE      ,keep_accents=_SCREAMING_SNAKE_CASE      ,bos_token=_SCREAMING_SNAKE_CASE      ,eos_token=_SCREAMING_SNAKE_CASE      ,unk_token=_SCREAMING_SNAKE_CASE      ,sep_token=_SCREAMING_SNAKE_CASE      ,pad_token=_SCREAMING_SNAKE_CASE      ,cls_token=_SCREAMING_SNAKE_CASE      ,mask_token=_SCREAMING_SNAKE_CASE      ,additional_special_tokens=_SCREAMING_SNAKE_CASE      ,**_SCREAMING_SNAKE_CASE      ,)

          _snake_case								=   3
          _snake_case								=   do_lower_case
          _snake_case								=   remove_space
          _snake_case								=   keep_accents
          _snake_case								=   vocab_file
          _snake_case								=   False if not self.vocab_file else True

          try:
               import jieba
          except ModuleNotFoundError as error:
               raise error.__class__(
                   "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
                   "See https://pypi.org/project/jieba/ for installation."					)
          _snake_case								=   jieba
          _snake_case								=   str.maketrans(" \n"      ,"\u2582\u2583"					)






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE = None					)							->    List[int]:
          _snake_case								=   [self.sep_token_id]
          _snake_case								=   [self.cls_token_id]
          if token_ids_a is None:
               return token_ids_a + sep + cls
          return token_ids_a + sep + token_ids_a + sep + cls






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE = None					)							->    List[int]:
          _snake_case								=   [self.sep_token_id]
          _snake_case								=   [2]

          if token_ids_a is None:
               return len(token_ids_a + sep					) * [0] + cls_segment_id
          return len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1] + cls_segment_id






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE = None					)							->    Tuple[str]:






          if not self.can_save_slow_tokenizer:
               raise ValueError(
                   "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                   "tokenizer."					)

          if not os.path.isdir(_SCREAMING_SNAKE_CASE					):
               logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
               return
          _snake_case								=   os.path.join(
              _SCREAMING_SNAKE_CASE      ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)

          if os.path.abspath(self.vocab_file					) != os.path.abspath(_SCREAMING_SNAKE_CASE					):
               copyfile(self.vocab_file      ,_SCREAMING_SNAKE_CASE					)

          return (out_vocab_file,)






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE      ,*_SCREAMING_SNAKE_CASE      ,**_SCREAMING_SNAKE_CASE					)							->    Dict:
          _snake_case								=   [
              " ".join([x.translate(self.translator					) for x in self.jieba.cut(_SCREAMING_SNAKE_CASE      ,cut_all=_SCREAMING_SNAKE_CASE					)]					)
              for text in batch_text_or_text_pairs
          ]
          return super()._batch_encode_plus(_SCREAMING_SNAKE_CASE      ,*_SCREAMING_SNAKE_CASE      ,**_SCREAMING_SNAKE_CASE					)






     def        _lowercase			(				self      ,*_SCREAMING_SNAKE_CASE      ,**_SCREAMING_SNAKE_CASE					)							->    str:
          _snake_case								=   super()._decode(*_SCREAMING_SNAKE_CASE      ,**_SCREAMING_SNAKE_CASE					)
          _snake_case								=   text.replace(" "      ,""					).replace("\u2582"      ," "					).replace("\u2583"      ,"\n"					)
          return text








'''simple docstring'''





from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


UpperCamelCase_		:    Optional[int]									=		logging.get_logger(__name__)

UpperCamelCase_		:    str									=		{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_		:    List[str]									=		{
    '''vocab_file''': {
        '''openai/clip-vit-base-patch32''': '''https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''openai/clip-vit-base-patch32''': '''https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''openai/clip-vit-base-patch32''': (
            '''https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase_		:    List[Any]									=		{
    '''openai/clip-vit-base-patch32''': 77,
}

class      _a       (		__lowerCAmelCase							):
     SCREAMING_SNAKE_CASE_							:					Any					       =		VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE_							:					List[Any]					       =		PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE_							:					Dict					       =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE_							:					int					       =		["""input_ids""", """attention_mask"""]
     SCREAMING_SNAKE_CASE_							:					Optional[Any]					       =		CLIPTokenizer






     def __init__(				self      ,_SCREAMING_SNAKE_CASE=None      ,_SCREAMING_SNAKE_CASE=None      ,_SCREAMING_SNAKE_CASE=None      ,_SCREAMING_SNAKE_CASE="<|endoftext|>"      ,_SCREAMING_SNAKE_CASE="<|startoftext|>"      ,_SCREAMING_SNAKE_CASE="<|endoftext|>"      ,_SCREAMING_SNAKE_CASE="<|endoftext|>"      ,**_SCREAMING_SNAKE_CASE      ,)							->    List[Any]:
          super().__init__(
              _SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE      ,tokenizer_file=_SCREAMING_SNAKE_CASE      ,unk_token=_SCREAMING_SNAKE_CASE      ,bos_token=_SCREAMING_SNAKE_CASE      ,eos_token=_SCREAMING_SNAKE_CASE      ,pad_token=_SCREAMING_SNAKE_CASE      ,**_SCREAMING_SNAKE_CASE      ,)

          if not isinstance(self.backend_tokenizer.pre_tokenizer      ,pre_tokenizers.Sequence					):
               raise ValueError(
                   "The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been"
                   " heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using"
                   " to be compatible with this version.The easiest way to do so is"
                   " `CLIPTokenizerFast.from_pretrained(\"path_to_local_folder_or_hub_repo, from_slow=True)`. If you want"
                   " to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of"
                   " transformers."					)

          self._wrap_decode_method_backend_tokenizer()






     def        _lowercase			(				self					)							->    Tuple:
          _snake_case							=   self.backend_tokenizer.decode

          def new_decode_method(*_SCREAMING_SNAKE_CASE      ,**_SCREAMING_SNAKE_CASE					):
               _snake_case							=   orig_decode_method(*_SCREAMING_SNAKE_CASE      ,**_SCREAMING_SNAKE_CASE					)
               _snake_case							=   text.replace(self.backend_tokenizer.model.end_of_word_suffix      ," "					).strip()
               return text

          _snake_case							=   new_decode_method






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE = None					)							->    List[int]:
          _snake_case							=   [self.bos_token_id]
          _snake_case							=   [self.eos_token_id]

          if token_ids_a is None:
               return bos_token + token_ids_a + eos_token
          return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE = None					)							->    List[int]:
          _snake_case							=   [self.bos_token_id]
          _snake_case							=   [self.eos_token_id]

          if token_ids_a is None:
               return len(bos_token + token_ids_a + eos_token					) * [0]
          return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token					) * [0]






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE = None					)							->    Tuple[str]:
          _snake_case							=   self._tokenizer.model.save(_SCREAMING_SNAKE_CASE      ,name=_SCREAMING_SNAKE_CASE					)
          return tuple(_SCREAMING_SNAKE_CASE					)


