'''simple docstring'''




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_UpperCamelCase				:		List[Any]       			=		logging.get_logger(__name__)

_UpperCamelCase				:		Optional[Any]       			=		{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_UpperCamelCase				:		Optional[Any]       			=		{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

_UpperCamelCase				:		Optional[Any]       			=		{
    'openai-gpt': 5_12,
}



def 						__snake_case (       lowerCAmelCase       :     Any			):
 __UpperCAmelCase         =					text.strip()
 if not text:
  return []
 __UpperCAmelCase         =					text.split()
 return tokens

class   _lowercase(     _lowerCamelCase				):



   """simple docstring"""





   def __init__(							self:    str			,a:    Optional[Any]=True			,a:    Dict=None			,a:    List[Any]=True			,a:    str=None			,a:    int=True			,):
    if never_split is None:
     __UpperCAmelCase         =					[]
    __UpperCAmelCase         =					do_lower_case
    __UpperCAmelCase         =					set(a	)
    __UpperCAmelCase         =					tokenize_chinese_chars
    __UpperCAmelCase         =					strip_accents
    __UpperCAmelCase         =					do_split_on_punc





   def 			snake_case	(							self:    Tuple			,a:    Optional[Any]			,a:    Optional[Any]=None	):
    __UpperCAmelCase         =					self.never_split.union(set(a	)	) if never_split else self.never_split
    __UpperCAmelCase         =					self._clean_text(a	)

    # This was added on November 1st, 2018 for the multilingual and Chinese
    # models. This is also applied to the English models now, but it doesn't
    # matter since the English models were not trained on any Chinese data
    # and generally don't have any Chinese data in them (there are Chinese
    # characters in the vocabulary because Wikipedia does have some Chinese
    # words in the English Wikipedia.).
    if self.tokenize_chinese_chars:
     __UpperCAmelCase         =					self._tokenize_chinese_chars(a	)
    # prevents treating the same character with different unicode codepoints as different characters
    __UpperCAmelCase         =					unicodedata.normalize('NFC'			,a	)
    __UpperCAmelCase         =					whitespace_tokenize(a	)
    __UpperCAmelCase         =					[]
    for token in orig_tokens:
     if token not in never_split:
      if self.do_lower_case:
       __UpperCAmelCase         =					token.lower()
       if self.strip_accents is not False:
        __UpperCAmelCase         =					self._run_strip_accents(a	)
      elif self.strip_accents:
       __UpperCAmelCase         =					self._run_strip_accents(a	)
     split_tokens.extend(self._run_split_on_punc(a			,a	)	)

    __UpperCAmelCase         =					whitespace_tokenize(' '.join(a	)	)
    return output_tokens





   def 			snake_case	(							self:    List[Any]			,a:    str	):
    __UpperCAmelCase         =					unicodedata.normalize('NFD'			,a	)
    __UpperCAmelCase         =					[]
    for char in text:
     __UpperCAmelCase         =					unicodedata.category(a	)
     if cat == "Mn":
      continue
     output.append(a	)
    return "".join(a	)





   def 			snake_case	(							self:    str			,a:    str			,a:    Union[str, Any]=None	):
    if not self.do_split_on_punc or (never_split is not None and text in never_split):
     return [text]
    __UpperCAmelCase         =					list(a	)
    __UpperCAmelCase         =					0
    __UpperCAmelCase         =					True
    __UpperCAmelCase         =					[]
    while i < len(a	):
     __UpperCAmelCase         =					chars[i]
     if _is_punctuation(a	):
      output.append([char]	)
      __UpperCAmelCase         =					True
     else:
      if start_new_word:
       output.append([]	)
      __UpperCAmelCase         =					False
      output[-1].append(a	)
     i += 1

    return ["".join(a	) for x in output]





   def 			snake_case	(							self:    List[Any]			,a:    str	):
    __UpperCAmelCase         =					[]
    for char in text:
     __UpperCAmelCase         =					ord(a	)
     if self._is_chinese_char(a	):
      output.append(' '	)
      output.append(a	)
      output.append(' '	)
     else:
      output.append(a	)
    return "".join(a	)





   def 			snake_case	(							self:    int			,a:    Optional[Any]	):
    # This defines a "chinese character" as anything in the CJK Unicode block:
    #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
    #
    # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
    # despite its name. The modern Korean Hangul alphabet is a different block,
    # as is Japanese Hiragana and Katakana. Those alphabets are used to write
    # space-separated words, so they are not treated specially and handled
    # like the all of the other languages.
    if (
        (cp >= 0x4e00 and cp <= 0x9fff)
        or (cp >= 0x3400 and cp <= 0x4dbf)  #
        or (cp >= 0x2_0000 and cp <= 0x2_a6df)  #
        or (cp >= 0x2_a700 and cp <= 0x2_b73f)  #
        or (cp >= 0x2_b740 and cp <= 0x2_b81f)  #
        or (cp >= 0x2_b820 and cp <= 0x2_ceaf)  #
        or (cp >= 0xf900 and cp <= 0xfaff)
        or (cp >= 0x2_f800 and cp <= 0x2_fa1f)  #
    ):  #
     return True

    return False





   def 			snake_case	(							self:    List[str]			,a:    List[str]	):
    __UpperCAmelCase         =					[]
    for char in text:
     __UpperCAmelCase         =					ord(a	)
     if cp == 0 or cp == 0xfffd or _is_control(a	):
      continue
     if _is_whitespace(a	):
      output.append(' '	)
     else:
      output.append(a	)
    return "".join(a	)




def 						__snake_case (       lowerCAmelCase       :     Optional[int]			):
 __UpperCAmelCase         =					set()
 __UpperCAmelCase         =					word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)			)
  __UpperCAmelCase         =					char
 return pairs



def 						__snake_case (       lowerCAmelCase       :     Union[str, Any]			):
 __UpperCAmelCase         =					text.replace('—'      ,  '-'			)
 __UpperCAmelCase         =					text.replace('–'      ,  '-'			)
 __UpperCAmelCase         =					text.replace('―'      ,  '-'			)
 __UpperCAmelCase         =					text.replace('…'      ,  '...'			)
 __UpperCAmelCase         =					text.replace('´'      ,  '\''			)
 __UpperCAmelCase         =					re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'      ,  r' \1 '      ,  lowerCAmelCase			)
 __UpperCAmelCase         =					re.sub(r'\s*\n\s*'      ,  ' \n '      ,  lowerCAmelCase			)
 __UpperCAmelCase         =					re.sub(r'[^\S\n]+'      ,  ' '      ,  lowerCAmelCase			)
 return text.strip()

class   _lowercase(     _lowerCamelCase				):



   """simple docstring"""





   __lowerCamelCase    =  VOCAB_FILES_NAMES
   __lowerCamelCase    =  PRETRAINED_VOCAB_FILES_MAP
   __lowerCamelCase    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __lowerCamelCase    =  ['''input_ids''', '''attention_mask''']





   def __init__(							self:    Tuple			,a:    List[Any]			,a:    Optional[Any]			,a:    Dict="<unk>"			,**a:    List[str]	):
    super().__init__(unk_token=a			,**a	)

    try:
     import ftfy
     from spacy.lang.en import English

     __UpperCAmelCase         =					English()
     __UpperCAmelCase         =					_nlp.tokenizer
     __UpperCAmelCase         =					ftfy.fix_text
    except ImportError:
     logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'	)
     __UpperCAmelCase         =					BasicTokenizer(do_lower_case=a	)
     __UpperCAmelCase         =					None

    with open(a			,encoding='utf-8'	) as vocab_handle:
     __UpperCAmelCase         =					json.load(a	)
    __UpperCAmelCase         =					{v: k for k, v in self.encoder.items()}
    with open(a			,encoding='utf-8'	) as merges_handle:
     __UpperCAmelCase         =					merges_handle.read().split('\n'	)[1:-1]
    __UpperCAmelCase         =					[tuple(merge.split()	) for merge in merges]
    __UpperCAmelCase         =					dict(zip(a			,range(len(a	)	)	)	)
    __UpperCAmelCase         =					{}





   @property
   def 			snake_case	(							self:    Dict	):
    return True





   @property
   def 			snake_case	(							self:    Dict	):
    return len(self.encoder	)





   def 			snake_case	(							self:    Dict	):
    return dict(self.encoder			,**self.added_tokens_encoder	)





   def 			snake_case	(							self:    Union[str, Any]			,a:    str	):
    __UpperCAmelCase         =					tuple(token[:-1]	) + (token[-1] + '</w>',)
    if token in self.cache:
     return self.cache[token]
    __UpperCAmelCase         =					get_pairs(a	)

    if not pairs:
     return token + "</w>"

    while True:
     __UpperCAmelCase         =					min(a			,key=lambda a	: self.bpe_ranks.get(a			,float('inf'	)	)	)
     if bigram not in self.bpe_ranks:
      break
     __UpperCAmelCase  ,			__UpperCAmelCase         =					bigram
     __UpperCAmelCase         =					[]
     __UpperCAmelCase         =					0
     while i < len(a	):
      try:
       __UpperCAmelCase         =					word.index(a			,a	)
      except ValueError:
       new_word.extend(word[i:]	)
       break
      else:
       new_word.extend(word[i:j]	)
       __UpperCAmelCase         =					j

      if word[i] == first and i < len(a	) - 1 and word[i + 1] == second:
       new_word.append(first + second	)
       i += 2
      else:
       new_word.append(word[i]	)
       i += 1
     __UpperCAmelCase         =					tuple(a	)
     __UpperCAmelCase         =					new_word
     if len(a	) == 1:
      break
     else:
      __UpperCAmelCase         =					get_pairs(a	)
    __UpperCAmelCase         =					' '.join(a	)
    if word == "\n  </w>":
     __UpperCAmelCase         =					'\n</w>'
    __UpperCAmelCase         =					word
    return word





   def 			snake_case	(							self:    Dict			,a:    Optional[int]	):
    __UpperCAmelCase         =					[]
    if self.fix_text is None:
     # Using BERT's BasicTokenizer
     __UpperCAmelCase         =					self.nlp.tokenize(a	)
     for token in text:
      split_tokens.extend(list(self.bpe(a	).split(' '	)	)	)
    else:
     # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
     __UpperCAmelCase         =					self.nlp(text_standardize(self.fix_text(a	)	)	)
     for token in text:
      split_tokens.extend(list(self.bpe(token.text.lower()	).split(' '	)	)	)
    return split_tokens





   def 			snake_case	(							self:    Dict			,a:    List[Any]	):
    return self.encoder.get(a			,self.encoder.get(self.unk_token	)	)





   def 			snake_case	(							self:    Dict			,a:    Union[str, Any]	):
    return self.decoder.get(a			,self.unk_token	)





   def 			snake_case	(							self:    List[Any]			,a:    List[str]	):
    __UpperCAmelCase         =					''.join(a	).replace('</w>'			,' '	).strip()
    return out_string





   def 			snake_case	(							self:    Any			,a:    str			,a:    Optional[str] = None	):
    if not os.path.isdir(a	):
     logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""	)
     return
    __UpperCAmelCase         =					os.path.join(
        a			,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
    __UpperCAmelCase         =					os.path.join(
        a			,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']	)

    with open(a			,'w'			,encoding='utf-8'	) as f:
     f.write(json.dumps(self.encoder			,indent=2			,sort_keys=a			,ensure_ascii=a	) + '\n'	)

    __UpperCAmelCase         =					0
    with open(a			,'w'			,encoding='utf-8'	) as writer:
     writer.write('#version: 0.2\n'	)
     for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,key=lambda a	: kv[1]	):
      if index != token_index:
       logger.warning(
           f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
           ' Please check that the tokenizer is not corrupted!'	)
       __UpperCAmelCase         =					token_index
      writer.write(' '.join(a	) + '\n'	)
      index += 1

    return vocab_file, merge_file




'''simple docstring'''




import unittest
from pathlib import Path

from transformers import is_vision_available, load_tool
from transformers.testing_utils import get_tests_dir

from .test_tools_common import ToolTesterMixin


if is_vision_available():
     from PIL import Image

class   _lowercase(     unittest.TestCase   ,_lowerCamelCase				):



   """simple docstring"""





   def 			snake_case	(							self:    Optional[int]	):
    __UpperCAmelCase         =					load_tool('image-captioning'	)
    self.tool.setup()
    __UpperCAmelCase         =					load_tool('image-captioning'			,remote=a	)





   def 			snake_case	(							self:    List[Any]	):
    __UpperCAmelCase         =					Image.open(Path(get_tests_dir('fixtures/tests_samples/COCO'	)	) / '000000039769.png'	)
    __UpperCAmelCase         =					self.tool(a	)
    self.assertEqual(a			,'two cats sleeping on a couch'	)





   def 			snake_case	(							self:    List[Any]	):
    __UpperCAmelCase         =					Image.open(Path(get_tests_dir('fixtures/tests_samples/COCO'	)	) / '000000039769.png'	)
    __UpperCAmelCase         =					self.remote_tool(a	)
    self.assertEqual(a			,'two cats sleeping on a couch'	)





   def 			snake_case	(							self:    Optional[Any]	):
    __UpperCAmelCase         =					Image.open(Path(get_tests_dir('fixtures/tests_samples/COCO'	)	) / '000000039769.png'	)
    __UpperCAmelCase         =					self.tool(image=a	)
    self.assertEqual(a			,'two cats sleeping on a couch'	)





   def 			snake_case	(							self:    Tuple	):
    __UpperCAmelCase         =					Image.open(Path(get_tests_dir('fixtures/tests_samples/COCO'	)	) / '000000039769.png'	)
    __UpperCAmelCase         =					self.remote_tool(image=a	)
    self.assertEqual(a			,'two cats sleeping on a couch'	)



