
'''simple docstring'''





import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


SCREAMING_SNAKE_CASE    =					logging.get_logger(__name__)

SCREAMING_SNAKE_CASE    =					{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

SCREAMING_SNAKE_CASE    =					{
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

SCREAMING_SNAKE_CASE    =					{
    """openai-gpt""": 5_1_2,
}







def snake_case_    (				lowercase__		):
  UpperCAmelCase__			:				Any												=       text.strip()
  if not text:
    return []
  UpperCAmelCase__			:				Dict												=       text.split()
  return tokens






class        UpperCAmelCase_       (						A							):

 '''simple docstring'''







 def __init__(				self		:     Union[str, Any] ,							snake_case__		:     Dict=True ,							snake_case__		:     str=None ,							snake_case__		:     Dict=True ,							snake_case__		:     Dict=None ,							snake_case__		:     Optional[int]=True ,							):



   '''simple docstring'''




   if never_split is None:
     UpperCAmelCase__			:				str												=       []
   UpperCAmelCase__			:				Dict												=       do_lower_case
   UpperCAmelCase__			:				Tuple												=       set(snake_case__   )
   UpperCAmelCase__			:				Optional[Any]												=       tokenize_chinese_chars
   UpperCAmelCase__			:				Union[str, Any]												=       strip_accents
   UpperCAmelCase__			:				Union[str, Any]												=       do_split_on_punc







 def        UpperCamelCase     (				self		:     List[str] ,							snake_case__		:     List[str] ,							snake_case__		:     int=None   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[Any]												=       self.never_split.union(set(snake_case__   )   ) if never_split else self.never_split
   UpperCAmelCase__			:				Union[str, Any]												=       self._clean_text(snake_case__   )

   # This was added on November 1st, 2018 for the multilingual and Chinese
   # models. This is also applied to the English models now, but it doesn't
   # matter since the English models were not trained on any Chinese data
   # and generally don't have any Chinese data in them (there are Chinese
   # characters in the vocabulary because Wikipedia does have some Chinese
   # words in the English Wikipedia.).
   if self.tokenize_chinese_chars:
     UpperCAmelCase__			:				Tuple												=       self._tokenize_chinese_chars(snake_case__   )
   # prevents treating the same character with different unicode codepoints as different characters
   UpperCAmelCase__			:				Union[str, Any]												=       unicodedata.normalize("NFC" ,							snake_case__   )
   UpperCAmelCase__			:				Any												=       whitespace_tokenize(snake_case__   )
   UpperCAmelCase__			:				List[str]												=       []
   for token in orig_tokens:
     if token not in never_split:
       if self.do_lower_case:
         UpperCAmelCase__			:				List[Any]												=       token.lower()
         if self.strip_accents is not False:
           UpperCAmelCase__			:				Any												=       self._run_strip_accents(snake_case__   )
       elif self.strip_accents:
         UpperCAmelCase__			:				Union[str, Any]												=       self._run_strip_accents(snake_case__   )
     split_tokens.extend(self._run_split_on_punc(snake_case__ ,							snake_case__   )   )

   UpperCAmelCase__			:				Any												=       whitespace_tokenize(" ".join(snake_case__   )   )
   return output_tokens







 def        UpperCamelCase     (				self		:     int ,							snake_case__		:     Any   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[int]												=       unicodedata.normalize("NFD" ,							snake_case__   )
   UpperCAmelCase__			:				Optional[int]												=       []
   for char in text:
     UpperCAmelCase__			:				Any												=       unicodedata.category(snake_case__   )
     if cat == "Mn":
       continue
     output.append(snake_case__   )
   return "".join(snake_case__   )







 def        UpperCamelCase     (				self		:     Dict ,							snake_case__		:     int ,							snake_case__		:     Tuple=None   ):



   '''simple docstring'''




   if not self.do_split_on_punc or (never_split is not None and text in never_split):
     return [text]
   UpperCAmelCase__			:				List[str]												=       list(snake_case__   )
   UpperCAmelCase__			:				Tuple												=       0
   UpperCAmelCase__			:				Union[str, Any]												=       True
   UpperCAmelCase__			:				Optional[int]												=       []
   while i < len(snake_case__   ):
     UpperCAmelCase__			:				List[Any]												=       chars[i]
     if _is_punctuation(snake_case__   ):
       output.append([char]   )
       UpperCAmelCase__			:				Any												=       True
     else:
       if start_new_word:
         output.append([]   )
       UpperCAmelCase__			:				Any												=       False
       output[-1].append(snake_case__   )
     i += 1

   return ["".join(snake_case__   ) for x in output]







 def        UpperCamelCase     (				self		:     Dict ,							snake_case__		:     List[str]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Optional[int]												=       []
   for char in text:
     UpperCAmelCase__			:				Optional[int]												=       ord(snake_case__   )
     if self._is_chinese_char(snake_case__   ):
       output.append(" "   )
       output.append(snake_case__   )
       output.append(" "   )
     else:
       output.append(snake_case__   )
   return "".join(snake_case__   )







 def        UpperCamelCase     (				self		:     Optional[Any] ,							snake_case__		:     Optional[Any]   ):



   '''simple docstring'''




   if (
       (cp >= 0x4E00 and cp <= 0x9FFF)
       or (cp >= 0x3400 and cp <= 0x4DBF)  #
       or (cp >= 0x20000 and cp <= 0x2A6DF)  #
       or (cp >= 0x2A700 and cp <= 0x2B73F)  #
       or (cp >= 0x2B740 and cp <= 0x2B81F)  #
       or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
       or (cp >= 0xF900 and cp <= 0xFAFF)
       or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
   ):  #
     return True

   return False







 def        UpperCamelCase     (				self		:     Optional[int] ,							snake_case__		:     Dict   ):



   '''simple docstring'''




   UpperCAmelCase__			:				List[Any]												=       []
   for char in text:
     UpperCAmelCase__			:				Tuple												=       ord(snake_case__   )
     if cp == 0 or cp == 0xFFFD or _is_control(snake_case__   ):
       continue
     if _is_whitespace(snake_case__   ):
       output.append(" "   )
     else:
       output.append(snake_case__   )
   return "".join(snake_case__   )







def snake_case_    (				lowercase__		):
  UpperCAmelCase__			:				Any												=       set()
  UpperCAmelCase__			:				Optional[int]												=       word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)		)
    UpperCAmelCase__			:				Any												=       char
  return pairs







def snake_case_    (				lowercase__		):
  UpperCAmelCase__			:				Union[str, Any]												=       text.replace("—"      ,					"-"		)
  UpperCAmelCase__			:				List[str]												=       text.replace("–"      ,					"-"		)
  UpperCAmelCase__			:				int												=       text.replace("―"      ,					"-"		)
  UpperCAmelCase__			:				str												=       text.replace("…"      ,					"..."		)
  UpperCAmelCase__			:				Any												=       text.replace("´"      ,					"'"		)
  UpperCAmelCase__			:				Dict												=       re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"      ,					R" \1 "      ,					lowercase__		)
  UpperCAmelCase__			:				Dict												=       re.sub(R"\s*\n\s*"      ,					" \n "      ,					lowercase__		)
  UpperCAmelCase__			:				int												=       re.sub(R"[^\S\n]+"      ,					" "      ,					lowercase__		)
  return text.strip()






class        UpperCAmelCase_       (						A							):

 '''simple docstring'''
 lowercase_				:						Tuple				=       VOCAB_FILES_NAMES
 lowercase_				:						Dict				=       PRETRAINED_VOCAB_FILES_MAP
 lowercase_				:						Dict				=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase_				:						List[Any]				=       ["input_ids", "attention_mask"]







 def __init__(				self		:     Union[str, Any] ,							snake_case__		:     Optional[Any] ,							snake_case__		:     List[str] ,							snake_case__		:     Union[str, Any]="<unk>" ,							**snake_case__		:     List[str]   ):



   '''simple docstring'''




   super().__init__(unk_token=snake_case__ ,							**snake_case__   )

   try:
     import ftfy
     from spacy.lang.en import English

     UpperCAmelCase__			:				List[Any]												=       English()
     UpperCAmelCase__			:				Optional[int]												=       _nlp.tokenizer
     UpperCAmelCase__			:				List[str]												=       ftfy.fix_text
   except ImportError:
     logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."   )
     UpperCAmelCase__			:				Union[str, Any]												=       BasicTokenizer(do_lower_case=snake_case__   )
     UpperCAmelCase__			:				Tuple												=       None

   with open(snake_case__ ,							encoding="utf-8"   ) as vocab_handle:
     UpperCAmelCase__			:				Dict												=       json.load(snake_case__   )
   UpperCAmelCase__			:				Tuple												=       {v: k for k, v in self.encoder.items()}
   with open(snake_case__ ,							encoding="utf-8"   ) as merges_handle:
     UpperCAmelCase__			:				List[Any]												=       merges_handle.read().split("\n"   )[1:-1]
   UpperCAmelCase__			:				Optional[int]												=       [tuple(merge.split()   ) for merge in merges]
   UpperCAmelCase__			:				int												=       dict(zip(snake_case__ ,							range(len(snake_case__   )   )   )   )
   UpperCAmelCase__			:				List[Any]												=       {}







 @property
 def        UpperCamelCase     (				self		:     Any   ):



   '''simple docstring'''




   return True







 @property
 def        UpperCamelCase     (				self		:     List[str]   ):



   '''simple docstring'''




   return len(self.encoder   )







 def        UpperCamelCase     (				self		:     Union[str, Any]   ):



   '''simple docstring'''




   return dict(self.encoder ,							**self.added_tokens_encoder   )







 def        UpperCamelCase     (				self		:     Union[str, Any] ,							snake_case__		:     int   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Any												=       tuple(token[:-1]   ) + (token[-1] + "</w>",)
   if token in self.cache:
     return self.cache[token]
   UpperCAmelCase__			:				List[str]												=       get_pairs(snake_case__   )

   if not pairs:
     return token + "</w>"

   while True:
     UpperCAmelCase__			:				List[str]												=       min(snake_case__ ,							key=lambda snake_case__   : self.bpe_ranks.get(snake_case__ ,							float("inf"   )   )   )
     if bigram not in self.bpe_ranks:
       break
     UpperCAmelCase__					, UpperCAmelCase__			:				int												=       bigram
     UpperCAmelCase__			:				str												=       []
     UpperCAmelCase__			:				int												=       0
     while i < len(snake_case__   ):
       try:
         UpperCAmelCase__			:				Tuple												=       word.index(snake_case__ ,							snake_case__   )
       except ValueError:
         new_word.extend(word[i:]   )
         break
       else:
         new_word.extend(word[i:j]   )
         UpperCAmelCase__			:				str												=       j

       if word[i] == first and i < len(snake_case__   ) - 1 and word[i + 1] == second:
         new_word.append(first + second   )
         i += 2
       else:
         new_word.append(word[i]   )
         i += 1
     UpperCAmelCase__			:				Any												=       tuple(snake_case__   )
     UpperCAmelCase__			:				Any												=       new_word
     if len(snake_case__   ) == 1:
       break
     else:
       UpperCAmelCase__			:				int												=       get_pairs(snake_case__   )
   UpperCAmelCase__			:				Tuple												=       " ".join(snake_case__   )
   if word == "\n  </w>":
     UpperCAmelCase__			:				Optional[int]												=       "\n</w>"
   UpperCAmelCase__			:				List[Any]												=       word
   return word







 def        UpperCamelCase     (				self		:     Any ,							snake_case__		:     Optional[int]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Tuple												=       []
   if self.fix_text is None:
     # Using BERT's BasicTokenizer
     UpperCAmelCase__			:				str												=       self.nlp.tokenize(snake_case__   )
     for token in text:
       split_tokens.extend(list(self.bpe(snake_case__   ).split(" "   )   )   )
   else:
     # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
     UpperCAmelCase__			:				Tuple												=       self.nlp(text_standardize(self.fix_text(snake_case__   )   )   )
     for token in text:
       split_tokens.extend(list(self.bpe(token.text.lower()   ).split(" "   )   )   )
   return split_tokens







 def        UpperCamelCase     (				self		:     str ,							snake_case__		:     Dict   ):



   '''simple docstring'''




   return self.encoder.get(snake_case__ ,							self.encoder.get(self.unk_token   )   )







 def        UpperCamelCase     (				self		:     Tuple ,							snake_case__		:     str   ):



   '''simple docstring'''




   return self.decoder.get(snake_case__ ,							self.unk_token   )







 def        UpperCamelCase     (				self		:     int ,							snake_case__		:     Union[str, Any]   ):



   '''simple docstring'''




   UpperCAmelCase__			:				Dict												=       "".join(snake_case__   ).replace("</w>" ,							" "   ).strip()
   return out_string







 def        UpperCamelCase     (				self		:     Tuple ,							snake_case__		:     str ,							snake_case__		:     Optional[str] = None   ):



   '''simple docstring'''




   if not os.path.isdir(snake_case__   ):
     logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
     return
   UpperCAmelCase__			:				Optional[Any]												=       os.path.join(
       snake_case__ ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
   UpperCAmelCase__			:				Optional[int]												=       os.path.join(
       snake_case__ ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]   )

   with open(snake_case__ ,							"w" ,							encoding="utf-8"   ) as f:
     f.write(json.dumps(self.encoder ,							indent=2 ,							sort_keys=snake_case__ ,							ensure_ascii=snake_case__   ) + "\n"   )

   UpperCAmelCase__			:				Optional[int]												=       0
   with open(snake_case__ ,							"w" ,							encoding="utf-8"   ) as writer:
     writer.write("#version: 0.2\n"   )
     for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,							key=lambda snake_case__   : kv[1]   ):
       if index != token_index:
         logger.warning(
             F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
             " Please check that the tokenizer is not corrupted!"   )
         UpperCAmelCase__			:				Tuple												=       token_index
       writer.write(" ".join(snake_case__   ) + "\n"   )
       index += 1

   return vocab_file, merge_file





'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


SCREAMING_SNAKE_CASE    =					{
    """configuration_donut_swin""": ["""DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP""", """DonutSwinConfig"""],
    """processing_donut""": ["""DonutProcessor"""],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      SCREAMING_SNAKE_CASE    =					[
          """DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """DonutSwinModel""",
          """DonutSwinPreTrainedModel""",
      ]

try:
      if not is_vision_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      SCREAMING_SNAKE_CASE    =					["""DonutFeatureExtractor"""]
      SCREAMING_SNAKE_CASE    =					["""DonutImageProcessor"""]


if TYPE_CHECKING:
      from .configuration_donut_swin import DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, DonutSwinConfig
      from .processing_donut import DonutProcessor

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_donut_swin import (
                DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
                DonutSwinModel,
                DonutSwinPreTrainedModel,
            )

      try:
            if not is_vision_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .feature_extraction_donut import DonutFeatureExtractor
            from .image_processing_donut import DonutImageProcessor

else:
      import sys

      SCREAMING_SNAKE_CASE    =					_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



