






import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


_A				:							List[Any]							= get_logger(__name__)


if TYPE_CHECKING:
  from .builder import DatasetBuilder


def        __snake_case       (						lowerCAmelCase_   ,       lowerCAmelCase_ = None  )							->	Tuple:
    SCREAMING_SNAKE_CASE__       							= importlib.import_module(UpperCamelCase__  )

    # TODO(QL): always update the module to add subsequent new authentication
    if hasattr(UpperCamelCase__   ,       '''_patched_for_streaming'''  ) and module._patched_for_streaming:
        return

    def wrap_auth(lowerCAmelCase_  ):
        @wraps(UpperCamelCase__  )
        def wrapper(*lowerCAmelCase_   ,       **lowerCAmelCase_  ):
            return function(*UpperCamelCase__   ,       use_auth_token=UpperCamelCase__   ,       **UpperCamelCase__  )

        SCREAMING_SNAKE_CASE__       							= "wrap_auth"
        return wrapper

    # open files in a streaming fashion
    patch_submodule(UpperCamelCase__   ,       '''open'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    patch_submodule(UpperCamelCase__   ,       '''os.listdir'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    patch_submodule(UpperCamelCase__   ,       '''os.walk'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    patch_submodule(UpperCamelCase__   ,       '''glob.glob'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    # allow to navigate in remote zip files
    patch_submodule(UpperCamelCase__   ,       '''os.path.join'''   ,       UpperCamelCase__  ).start()
    patch_submodule(UpperCamelCase__   ,       '''os.path.dirname'''   ,       UpperCamelCase__  ).start()
    patch_submodule(UpperCamelCase__   ,       '''os.path.basename'''   ,       UpperCamelCase__  ).start()
    patch_submodule(UpperCamelCase__   ,       '''os.path.relpath'''   ,       UpperCamelCase__  ).start()
    patch_submodule(UpperCamelCase__   ,       '''os.path.split'''   ,       UpperCamelCase__  ).start()
    patch_submodule(UpperCamelCase__   ,       '''os.path.splitext'''   ,       UpperCamelCase__  ).start()
    # allow checks on paths
    patch_submodule(UpperCamelCase__   ,       '''os.path.exists'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    patch_submodule(UpperCamelCase__   ,       '''os.path.isdir'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    patch_submodule(UpperCamelCase__   ,       '''os.path.isfile'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    patch_submodule(UpperCamelCase__   ,       '''os.path.getsize'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    patch_submodule(UpperCamelCase__   ,       '''pathlib.Path'''   ,       UpperCamelCase__  ).start()
    # file readers
    patch_submodule(UpperCamelCase__   ,       '''gzip.open'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    patch_submodule(UpperCamelCase__   ,       '''numpy.load'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    patch_submodule(UpperCamelCase__   ,       '''pandas.read_csv'''   ,       wrap_auth(UpperCamelCase__  )   ,       attrs=['''__version__''']  ).start()
    patch_submodule(UpperCamelCase__   ,       '''pandas.read_excel'''   ,       wrap_auth(UpperCamelCase__  )   ,       attrs=['''__version__''']  ).start()
    patch_submodule(UpperCamelCase__   ,       '''scipy.io.loadmat'''   ,       wrap_auth(UpperCamelCase__  )   ,       attrs=['''__version__''']  ).start()
    patch_submodule(UpperCamelCase__   ,       '''xml.etree.ElementTree.parse'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    patch_submodule(UpperCamelCase__   ,       '''xml.dom.minidom.parse'''   ,       wrap_auth(UpperCamelCase__  )  ).start()
    SCREAMING_SNAKE_CASE__       							= True



def        __snake_case       (						lowerCAmelCase_  )							->	Any:
    extend_module_for_streaming(builder.__module__   ,       use_auth_token=builder.use_auth_token  )
    # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
    if not builder.__module__.startswith('''datasets.'''  ):  # check that it's not a packaged builder like csv
        for imports in get_imports(inspect.getfile(builder.__class__  )  ):
            if imports[0] == "internal":
                SCREAMING_SNAKE_CASE__       							= imports[1]
                SCREAMING_SNAKE_CASE__       							= ".".join(builder.__module__.split('''.'''  )[:-1] + [internal_import_name]  )
                extend_module_for_streaming(UpperCamelCase__   ,       use_auth_token=builder.use_auth_token  )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
    from .builder import DatasetBuilder

    SCREAMING_SNAKE_CASE__       							= [
        cls.__module__
        for cls in type(UpperCamelCase__  ).__mro__[1:]  # make sure it's not the same module we've already patched
        if issubclass(cls   ,       UpperCamelCase__  ) and cls.__module__ != DatasetBuilder.__module__
    ]  # check it's not a standard builder from datasets.builder
    for module in parent_builder_modules:
        extend_module_for_streaming(UpperCamelCase__   ,       use_auth_token=builder.use_auth_token  )







import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
  import transformer_engine.pytorch as te


def        __snake_case       (						lowerCAmelCase_   ,       lowerCAmelCase_=True   ,       lowerCAmelCase_=True   ,       lowerCAmelCase_=True  )							->	Tuple:
    if not is_fpa_available():
        raise ImportError('''Using `convert_model` requires transformer_engine to be installed.'''  )
    for name, module in model.named_children():
        if isinstance(lowerCAmelCase_   ,       nn.Linear  ) and to_transformer_engine and _convert_linear:
            # Return early if the linear layer weights are not multiples of 16
            if any(p % 1_6 != 0 for p in module.weight.shape  ):
                return
            SCREAMING_SNAKE_CASE__       							= module.bias is not None
            SCREAMING_SNAKE_CASE__       							= te.Linear(
                module.in_features   ,       module.out_features   ,       bias=lowerCAmelCase_   ,       params_dtype=module.weight.dtype  )
            SCREAMING_SNAKE_CASE__       							= module.weight.data.clone()
            if has_bias:
                SCREAMING_SNAKE_CASE__       							= module.bias.data.clone()

            setattr(lowerCAmelCase_   ,       lowerCAmelCase_   ,       lowerCAmelCase_  )
        elif isinstance(lowerCAmelCase_   ,       nn.LayerNorm  ) and to_transformer_engine and _convert_ln:
            SCREAMING_SNAKE_CASE__       							= te.LayerNorm(module.normalized_shape[0]   ,       eps=module.eps   ,       params_dtype=module.weight.dtype  )
            SCREAMING_SNAKE_CASE__       							= module.weight.data.clone()
            SCREAMING_SNAKE_CASE__       							= module.bias.data.clone()

            setattr(lowerCAmelCase_   ,       lowerCAmelCase_   ,       lowerCAmelCase_  )
        elif isinstance(lowerCAmelCase_   ,       te.Linear  ) and not to_transformer_engine and _convert_linear:
            SCREAMING_SNAKE_CASE__       							= module.bias is not None
            SCREAMING_SNAKE_CASE__       							= nn.Linear(
                module.in_features   ,       module.out_features   ,       bias=lowerCAmelCase_   ,       params_dtype=module.weight.dtype  )
            SCREAMING_SNAKE_CASE__       							= module.weight.data.clone()
            if has_bias:
                SCREAMING_SNAKE_CASE__       							= module.bias.data.clone()

            setattr(lowerCAmelCase_   ,       lowerCAmelCase_   ,       lowerCAmelCase_  )
        elif isinstance(lowerCAmelCase_   ,       te.LayerNorm  ) and not to_transformer_engine and _convert_ln:
            SCREAMING_SNAKE_CASE__       							= nn.LayerNorm(module.normalized_shape[0]   ,       eps=module.eps   ,       params_dtype=module.weight.dtype  )
            SCREAMING_SNAKE_CASE__       							= module.weight.data.clone()
            SCREAMING_SNAKE_CASE__       							= module.bias.data.clone()

            setattr(lowerCAmelCase_   ,       lowerCAmelCase_   ,       lowerCAmelCase_  )
        else:
            convert_model(
                lowerCAmelCase_   ,       to_transformer_engine=lowerCAmelCase_   ,       _convert_linear=_convert_linear   ,       _convert_ln=_convert_ln   ,       )



def        __snake_case       (						lowerCAmelCase_  )							->	Union[str, Any]:
    if not is_fpa_available():
        raise ImportError('''Using `has_transformer_engine_layers` requires transformer_engine to be installed.'''  )
    for m in model.modules():
        if isinstance(lowerCAmelCase_   ,       (te.LayerNorm, te.Linear)  ):
            return True
    return False
