






"""simple docstring"""




import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


_snake_case					   =     get_logger(__name__)


if TYPE_CHECKING:
 from .builder import DatasetBuilder

def     lowerCAmelCase__  (  UpperCamelCase__  ,     UpperCamelCase__ = None       ):



  '''simple docstring'''
  _a						: Any    =			importlib.import_module(UpperCamelCase__       )

  # TODO(QL): always update the module to add subsequent new authentication
  if hasattr(UpperCamelCase__  ,     """_patched_for_streaming"""       ) and module._patched_for_streaming:
    return

  def wrap_auth(UpperCamelCase__       ):
    @wraps(UpperCamelCase__       )
    def wrapper(*UpperCamelCase__  ,     **UpperCamelCase__       ):
      return function(*UpperCamelCase__  ,     use_auth_token=UpperCamelCase__  ,     **UpperCamelCase__       )

    _a						: Tuple    =			"""wrap_auth"""
    return wrapper

  # open files in a streaming fashion
  patch_submodule(UpperCamelCase__  ,     """open"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  patch_submodule(UpperCamelCase__  ,     """os.listdir"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  patch_submodule(UpperCamelCase__  ,     """os.walk"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  patch_submodule(UpperCamelCase__  ,     """glob.glob"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  # allow to navigate in remote zip files
  patch_submodule(UpperCamelCase__  ,     """os.path.join"""  ,     UpperCamelCase__       ).start()
  patch_submodule(UpperCamelCase__  ,     """os.path.dirname"""  ,     UpperCamelCase__       ).start()
  patch_submodule(UpperCamelCase__  ,     """os.path.basename"""  ,     UpperCamelCase__       ).start()
  patch_submodule(UpperCamelCase__  ,     """os.path.relpath"""  ,     UpperCamelCase__       ).start()
  patch_submodule(UpperCamelCase__  ,     """os.path.split"""  ,     UpperCamelCase__       ).start()
  patch_submodule(UpperCamelCase__  ,     """os.path.splitext"""  ,     UpperCamelCase__       ).start()
  # allow checks on paths
  patch_submodule(UpperCamelCase__  ,     """os.path.exists"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  patch_submodule(UpperCamelCase__  ,     """os.path.isdir"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  patch_submodule(UpperCamelCase__  ,     """os.path.isfile"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  patch_submodule(UpperCamelCase__  ,     """os.path.getsize"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  patch_submodule(UpperCamelCase__  ,     """pathlib.Path"""  ,     UpperCamelCase__       ).start()
  # file readers
  patch_submodule(UpperCamelCase__  ,     """gzip.open"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  patch_submodule(UpperCamelCase__  ,     """numpy.load"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  patch_submodule(UpperCamelCase__  ,     """pandas.read_csv"""  ,     wrap_auth(UpperCamelCase__       )  ,     attrs=["""__version__"""]       ).start()
  patch_submodule(UpperCamelCase__  ,     """pandas.read_excel"""  ,     wrap_auth(UpperCamelCase__       )  ,     attrs=["""__version__"""]       ).start()
  patch_submodule(UpperCamelCase__  ,     """scipy.io.loadmat"""  ,     wrap_auth(UpperCamelCase__       )  ,     attrs=["""__version__"""]       ).start()
  patch_submodule(UpperCamelCase__  ,     """xml.etree.ElementTree.parse"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  patch_submodule(UpperCamelCase__  ,     """xml.dom.minidom.parse"""  ,     wrap_auth(UpperCamelCase__       )       ).start()
  _a						: str    =			True



def     lowerCAmelCase__  (  UpperCamelCase__       ):



  '''simple docstring'''
  extend_module_for_streaming(builder.__module__  ,     use_auth_token=builder.use_auth_token       )
  # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
  if not builder.__module__.startswith("""datasets."""       ):  # check that it's not a packaged builder like csv
    for imports in get_imports(inspect.getfile(builder.__class__       )       ):
      if imports[0] == "internal":
        _a						: int    =			imports[1]
        _a						: Any    =			""".""".join(builder.__module__.split("""."""       )[:-1] + [internal_import_name]       )
        extend_module_for_streaming(UpperCamelCase__  ,     use_auth_token=builder.use_auth_token       )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
  from .builder import DatasetBuilder

  _a						: str    =			[
      cls.__module__
      for cls in type(UpperCamelCase__       ).__mro__[1:]  # make sure it's not the same module we've already patched
      if issubclass(cls  ,     UpperCamelCase__       ) and cls.__module__ != DatasetBuilder.__module__
  ]  # check it's not a standard builder from datasets.builder
  for module in parent_builder_modules:
    extend_module_for_streaming(UpperCamelCase__  ,     use_auth_token=builder.use_auth_token       )









"""simple docstring"""




# this script builds a small sample spm file tests/fixtures/test_sentencepiece_no_bos.model, with features needed by pegasus

# 1. pip install sentencepiece
#
# 2. wget https://raw.githubusercontent.com/google/sentencepiece/master/data/botchan.txt

# 3. build
import sentencepiece as spm

# pegasus:
# 1. no bos
# 2. eos_id is 1
# 3. unk_id is 2
# build a sample spm file accordingly
spm.SentencePieceTrainer.train(
    '--input=botchan.txt --model_prefix=test_sentencepiece_no_bos --bos_id=-1 --unk_id=2  --eos_id=1  --vocab_size=1000'
)

# 4. now update the fixture
# mv test_sentencepiece_no_bos.model ../../tests/fixtures/

