




import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__magic_name__     :     str  							=					logging.get_logger(__name__)
__magic_name__     :     Union[str, Any]  							=					{"""vocab_file""": """spiece.model"""}

__magic_name__     :     List[Any]  							=					{
    """vocab_file""": {
        """google/fnet-base""": """https://huggingface.co/google/fnet-base/resolve/main/spiece.model""",
        """google/fnet-large""": """https://huggingface.co/google/fnet-large/resolve/main/spiece.model""",
    },
}

__magic_name__     :     Any  							=					{
    """google/fnet-base""": 5_12,
    """google/fnet-large""": 5_12,
}

__magic_name__     :     Optional[int]  							=					"""▁"""





class 						SCREAMING_SNAKE_CASE__		(_SCREAMING_SNAKE_CASE ):
					lowercase_   :			Union[str, Any]   =   VOCAB_FILES_NAMES
					lowercase_   :			Optional[int]   =   PRETRAINED_VOCAB_FILES_MAP
					lowercase_   :			Optional[int]   =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase_   :			List[Any]   =   ["input_ids", "token_type_ids"]
					def __init__(	self  :		List[str]					,	__lowerCamelCase  :		Dict					,	__lowerCamelCase  :		int=False					,	__lowerCamelCase  :		Tuple=True					,	__lowerCamelCase  :		List[str]=True					,	__lowerCamelCase  :		int="<unk>"					,	__lowerCamelCase  :		Union[str, Any]="[SEP]"					,	__lowerCamelCase  :		int="<pad>"					,	__lowerCamelCase  :		str="[CLS]"					,	__lowerCamelCase  :		Union[str, Any]="[MASK]"					,	__lowerCamelCase  :		Optional[Any] = None					,	**__lowerCamelCase  :		Tuple					,	):



							"""simple docstring"""
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.
							lowerCAmelCase__								=			(
							    AddedToken(__A					,	lstrip=__A					,	rstrip=__A					,	normalized=__A	)
							    if isinstance(__A					,	__A	)
							    else mask_token
							)

							lowerCAmelCase__								=			{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=__A					,	remove_space=__A					,	keep_accents=__A					,	unk_token=__A					,	sep_token=__A					,	pad_token=__A					,	cls_token=__A					,	mask_token=__A					,	sp_model_kwargs=self.sp_model_kwargs					,	**__A					,	)

							lowerCAmelCase__								=			do_lower_case
							lowerCAmelCase__								=			remove_space
							lowerCAmelCase__								=			keep_accents
							lowerCAmelCase__								=			vocab_file

							lowerCAmelCase__								=			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
							self.sp_model.Load(__A	)
					@property
					def 					A__			(	self  :		Dict	):



							"""simple docstring"""
							return len(self.sp_model	)
					def 					A__			(	self  :		Optional[Any]	):



							"""simple docstring"""
							lowerCAmelCase__								=			{self.convert_ids_to_tokens(__A	): i for i in range(self.vocab_size	)}
							vocab.update(self.added_tokens_encoder	)
							return vocab
					def __getstate__(	self  :		Dict	):



							"""simple docstring"""
							lowerCAmelCase__								=			self.__dict__.copy()
							lowerCAmelCase__								=			None
							return state
					def __setstate__(	self  :		str					,	__lowerCamelCase  :		Tuple	):



							"""simple docstring"""
							lowerCAmelCase__								=			d

							# for backward compatibility
							if not hasattr(self					,	'''sp_model_kwargs'''	):
									lowerCAmelCase__								=			{}

							lowerCAmelCase__								=			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
							self.sp_model.Load(self.vocab_file	)
					def 					A__			(	self  :		Tuple					,	__lowerCamelCase  :		Optional[int]	):



							"""simple docstring"""
							if self.remove_space:
									lowerCAmelCase__								=			''' '''.join(inputs.strip().split()	)
							else:
									lowerCAmelCase__								=			inputs
							lowerCAmelCase__								=			outputs.replace('''``'''					,	'''"'''	).replace('''\'\''''					,	'''"'''	)

							if not self.keep_accents:
									lowerCAmelCase__								=			unicodedata.normalize('''NFKD'''					,	__A	)
									lowerCAmelCase__								=			''''''.join([c for c in outputs if not unicodedata.combining(__A	)]	)
							if self.do_lower_case:
									lowerCAmelCase__								=			outputs.lower()

							return outputs
					def 					A__			(	self  :		int					,	__lowerCamelCase  :		str	):



							"""simple docstring"""
							lowerCAmelCase__								=			self.preprocess_text(__A	)
							lowerCAmelCase__								=			self.sp_model.encode(__A					,	out_type=__A	)
							lowerCAmelCase__								=			[]
							for piece in pieces:
									if len(__A	) > 1 and piece[-1] == str(''','''	) and piece[-2].isdigit():
											lowerCAmelCase__								=			self.sp_model.EncodeAsPieces(piece[:-1].replace(__A					,	''''''	)	)
											if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
													if len(cur_pieces[0]	) == 1:
															lowerCAmelCase__								=			cur_pieces[1:]
													else:
															lowerCAmelCase__								=			cur_pieces[0][1:]
											cur_pieces.append(piece[-1]	)
											new_pieces.extend(__A	)
									else:
											new_pieces.append(__A	)

							return new_pieces
					def 					A__			(	self  :		Any					,	__lowerCamelCase  :		Union[str, Any]	):



							"""simple docstring"""
							return self.sp_model.PieceToId(__A	)
					def 					A__			(	self  :		str					,	__lowerCamelCase  :		List[Any]	):



							"""simple docstring"""
							return self.sp_model.IdToPiece(__A	)
					def 					A__			(	self  :		Union[str, Any]					,	__lowerCamelCase  :		Tuple	):



							"""simple docstring"""
							lowerCAmelCase__								=			[]
							lowerCAmelCase__								=			''''''
							lowerCAmelCase__								=			False
							for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
											if not prev_is_special:
													out_string += " "
											out_string += self.sp_model.decode(__A	) + token
											lowerCAmelCase__								=			True
											lowerCAmelCase__								=			[]
									else:
											current_sub_tokens.append(__A	)
											lowerCAmelCase__								=			False
							out_string += self.sp_model.decode(__A	)
							return out_string.strip()
					def 					A__			(	self  :		int					,	__lowerCamelCase  :		int					,	__lowerCamelCase  :		int = False					,	__lowerCamelCase  :		Dict = None					,	__lowerCamelCase  :		Any = True					,	**__lowerCamelCase  :		Optional[int]					,	):



							"""simple docstring"""
							lowerCAmelCase__								=			kwargs.pop('''use_source_tokenizer'''					,	__A	)

							lowerCAmelCase__								=			self.convert_ids_to_tokens(__A					,	skip_special_tokens=__A	)

							# To avoid mixing byte-level and unicode for byte-level BPT
							# we need to build string separately for added tokens and byte-level tokens
							# cf. https://github.com/huggingface/transformers/issues/1133
							lowerCAmelCase__								=			[]
							lowerCAmelCase__								=			[]
							for token in filtered_tokens:
									if skip_special_tokens and token in self.all_special_ids:
											continue
									if token in self.added_tokens_encoder:
											if current_sub_text:
													sub_texts.append(self.convert_tokens_to_string(__A	)	)
													lowerCAmelCase__								=			[]
											sub_texts.append(__A	)
									else:
											current_sub_text.append(__A	)
							if current_sub_text:
									sub_texts.append(self.convert_tokens_to_string(__A	)	)

							# Mimic the behavior of the Rust tokenizer:
							# No space after <unk>
							if spaces_between_special_tokens:
									lowerCAmelCase__								=			re.sub(r'''(<unk>) '''					,	r'''\1'''					,	''' '''.join(__A	)	)
							else:
									lowerCAmelCase__								=			''''''.join(__A	)

							lowerCAmelCase__								=			(
							    clean_up_tokenization_spaces
							    if clean_up_tokenization_spaces is not None
							    else self.clean_up_tokenization_spaces
							)
							if clean_up_tokenization_spaces:
									lowerCAmelCase__								=			self.clean_up_tokenization(__A	)
									return clean_text
							else:
									return text
					def 					A__			(	self  :		List[str]					,	__lowerCamelCase  :		Any					,	__lowerCamelCase  :		Tuple = None	):



							"""simple docstring"""
							lowerCAmelCase__								=			[self.sep_token_id]
							lowerCAmelCase__								=			[self.cls_token_id]
							if token_ids_a is None:
									return cls + token_ids_a + sep
							return cls + token_ids_a + sep + token_ids_a + sep
					def 					A__			(	self  :		Any					,	__lowerCamelCase  :		Union[str, Any]					,	__lowerCamelCase  :		List[Any] = None					,	__lowerCamelCase  :		Any = False	):



							"""simple docstring"""

							if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=__A					,	token_ids_a=__A					,	already_has_special_tokens=__A	)

							if token_ids_a is not None:
									return [1] + ([0] * len(__A	)) + [1] + ([0] * len(__A	)) + [1]
							return [1] + ([0] * len(__A	)) + [1]
					def 					A__			(	self  :		Tuple					,	__lowerCamelCase  :		str					,	__lowerCamelCase  :		Optional[int] = None	):



							"""simple docstring"""
							lowerCAmelCase__								=			[self.sep_token_id]
							lowerCAmelCase__								=			[self.cls_token_id]

							if token_ids_a is None:
									return len(cls + token_ids_a + sep	) * [0]
							return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]



					def 					A__			(	self  :		List[str]					,	__lowerCamelCase  :		List[Any]					,	__lowerCamelCase  :		Dict = None	):



							"""simple docstring"""
							if not os.path.isdir(__A	):
									logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""	)
									return
							lowerCAmelCase__								=			os.path.join(
							    __A					,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

							if os.path.abspath(self.vocab_file	) != os.path.abspath(__A	) and os.path.isfile(self.vocab_file	):
									copyfile(self.vocab_file					,	__A	)
							elif not os.path.isfile(self.vocab_file	):
									with open(__A					,	'''wb'''	) as fi:
											lowerCAmelCase__								=			self.sp_model.serialized_model_proto()
											fi.write(__A	)

							return (out_vocab_file,)







import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


__magic_name__     :     List[Any]  							=					get_logger(__name__)


if TYPE_CHECKING:
      from .builder import DatasetBuilder
def 		a_       (   __lowerCAmelCase      ,   __lowerCAmelCase = None							):
  lowerCAmelCase__								=			importlib.import_module(__lowerCAmelCase							)

  # TODO(QL): always update the module to add subsequent new authentication
  if hasattr(__lowerCAmelCase      ,   '''_patched_for_streaming'''							) and module._patched_for_streaming:
    return

  def wrap_auth(__lowerCAmelCase							):
    @wraps(__lowerCAmelCase							)
    def wrapper(*__lowerCAmelCase      ,   **__lowerCAmelCase							):
      return function(*__lowerCAmelCase      ,   use_auth_token=__lowerCAmelCase      ,   **__lowerCAmelCase							)

    lowerCAmelCase__								=			'''wrap_auth'''
    return wrapper

  # open files in a streaming fashion
  patch_submodule(__lowerCAmelCase      ,   '''open'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.listdir'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.walk'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''glob.glob'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  # allow to navigate in remote zip files
  patch_submodule(__lowerCAmelCase      ,   '''os.path.join'''      ,   __lowerCAmelCase							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.dirname'''      ,   __lowerCAmelCase							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.basename'''      ,   __lowerCAmelCase							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.relpath'''      ,   __lowerCAmelCase							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.split'''      ,   __lowerCAmelCase							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.splitext'''      ,   __lowerCAmelCase							).start()
  # allow checks on paths
  patch_submodule(__lowerCAmelCase      ,   '''os.path.exists'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.isdir'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.isfile'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.getsize'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''pathlib.Path'''      ,   __lowerCAmelCase							).start()
  # file readers
  patch_submodule(__lowerCAmelCase      ,   '''gzip.open'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''numpy.load'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''pandas.read_csv'''      ,   wrap_auth(__lowerCAmelCase							)      ,   attrs=['''__version__''']							).start()
  patch_submodule(__lowerCAmelCase      ,   '''pandas.read_excel'''      ,   wrap_auth(__lowerCAmelCase							)      ,   attrs=['''__version__''']							).start()
  patch_submodule(__lowerCAmelCase      ,   '''scipy.io.loadmat'''      ,   wrap_auth(__lowerCAmelCase							)      ,   attrs=['''__version__''']							).start()
  patch_submodule(__lowerCAmelCase      ,   '''xml.etree.ElementTree.parse'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''xml.dom.minidom.parse'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  lowerCAmelCase__								=			True




def 		a_       (   __lowerCAmelCase							):
  extend_module_for_streaming(builder.__module__      ,   use_auth_token=builder.use_auth_token							)
  # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
  if not builder.__module__.startswith('''datasets.'''							):  # check that it's not a packaged builder like csv
    for imports in get_imports(inspect.getfile(builder.__class__							)							):
      if imports[0] == "internal":
        lowerCAmelCase__								=			imports[1]
        lowerCAmelCase__								=			'''.'''.join(builder.__module__.split('''.'''							)[:-1] + [internal_import_name]							)
        extend_module_for_streaming(__lowerCAmelCase      ,   use_auth_token=builder.use_auth_token							)

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
  from .builder import DatasetBuilder

  lowerCAmelCase__								=			[
      cls.__module__
      for cls in type(__lowerCAmelCase							).__mro__[1:]  # make sure it's not the same module we've already patched
      if issubclass(cls      ,   __lowerCAmelCase							) and cls.__module__ != DatasetBuilder.__module__
  ]  # check it's not a standard builder from datasets.builder
  for module in parent_builder_modules:
    extend_module_for_streaming(__lowerCAmelCase      ,   use_auth_token=builder.use_auth_token							)

