


import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


snake_case__     							= get_logger(__name__)

# Datasets
snake_case__     							= 'https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets'
snake_case__     							= 'https://cdn-datasets.huggingface.co/datasets/datasets'
snake_case__     							= 'https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}'

# Metrics
snake_case__     							= 'https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics'
snake_case__     							= 'https://cdn-datasets.huggingface.co/datasets/metric'
snake_case__     							= 'https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}'

# Hub
snake_case__     							= os.environ.get('HF_ENDPOINT', 'https://huggingface.co')
snake_case__     							= HF_ENDPOINT + '/datasets/{repo_id}/resolve/{revision}/{path}'
snake_case__     							= 'main'

snake_case__     							= version.parse(platform.python_version())

if PY_VERSION < version.parse('3.8'):
			import importlib_metadata
else:
			import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
snake_case__     							= {'1', 'ON', 'YES', 'TRUE'}
snake_case__     							= ENV_VARS_TRUE_VALUES.union({'AUTO'})


# Imports
snake_case__     							= version.parse(importlib_metadata.version('dill'))
snake_case__     							= version.parse(importlib_metadata.version('pandas'))
snake_case__     							= version.parse(importlib_metadata.version('pyarrow'))

snake_case__     							= os.environ.get('USE_TF', 'AUTO').upper()
snake_case__     							= os.environ.get('USE_TORCH', 'AUTO').upper()
snake_case__     							= os.environ.get('USE_JAX', 'AUTO').upper()

snake_case__     							= 'N/A'
snake_case__     							= False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
			snake_case__     							= importlib.util.find_spec('torch') is not None
			if TORCH_AVAILABLE:
						try:
									snake_case__     							= version.parse(importlib_metadata.version('torch'))
									logger.info(F"""PyTorch version {TORCH_VERSION} available.""")
						except importlib_metadata.PackageNotFoundError:
									pass
else:
			logger.info('Disabling PyTorch because USE_TF is set')

snake_case__     							= 'N/A'
snake_case__     							= False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
			snake_case__     							= importlib.util.find_spec('tensorflow') is not None
			if TF_AVAILABLE:
						# For the metadata, we have to look for both tensorflow and tensorflow-cpu
						for package in [
						    "tensorflow",
						    "tensorflow-cpu",
						    "tensorflow-gpu",
						    "tf-nightly",
						    "tf-nightly-cpu",
						    "tf-nightly-gpu",
						    "intel-tensorflow",
						    "tensorflow-rocm",
						    "tensorflow-macos",
						]:
									try:
												snake_case__     							= version.parse(importlib_metadata.version(package))
									except importlib_metadata.PackageNotFoundError:
												continue
									else:
												break
						else:
									snake_case__     							= False
			if TF_AVAILABLE:
						if TF_VERSION.major < 2:
									logger.info(F"""TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.""")
									snake_case__     							= False
						else:
									logger.info(F"""TensorFlow version {TF_VERSION} available.""")
else:
			logger.info('Disabling Tensorflow because USE_TORCH is set')


snake_case__     							= 'N/A'
snake_case__     							= False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
			snake_case__     							= importlib.util.find_spec('jax') is not None and importlib.util.find_spec('jaxlib') is not None
			if JAX_AVAILABLE:
						try:
									snake_case__     							= version.parse(importlib_metadata.version('jax'))
									logger.info(F"""JAX version {JAX_VERSION} available.""")
						except importlib_metadata.PackageNotFoundError:
									pass
else:
			logger.info('Disabling JAX because USE_JAX is set to False')


snake_case__     							= os.environ.get('USE_BEAM', 'AUTO').upper()
snake_case__     							= 'N/A'
snake_case__     							= False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
			try:
						snake_case__     							= version.parse(importlib_metadata.version('apache_beam'))
						snake_case__     							= True
						logger.info(F"""Apache Beam version {BEAM_VERSION} available.""")
			except importlib_metadata.PackageNotFoundError:
						pass
else:
			logger.info('Disabling Apache Beam because USE_BEAM is set to False')


# Optional tools for data loading
snake_case__     							= importlib.util.find_spec('sqlalchemy') is not None

# Optional tools for feature decoding
snake_case__     							= importlib.util.find_spec('PIL') is not None
snake_case__     							= importlib.util.find_spec('soundfile') is not None and version.parse(
    importlib.import_module('soundfile').__libsndfile_version__
) >= version.parse('1.0.31')
snake_case__     							= importlib.util.find_spec('soundfile') is not None and version.parse(
    importlib.import_module('soundfile').__libsndfile_version__
) >= version.parse('1.1.0')

# Optional compression tools
snake_case__     							= importlib.util.find_spec('rarfile') is not None
snake_case__     							= importlib.util.find_spec('zstandard') is not None
snake_case__     							= importlib.util.find_spec('lz4') is not None
snake_case__     							= importlib.util.find_spec('py7zr') is not None


# Cache location
snake_case__     							= '~/.cache'
snake_case__     							= os.getenv('XDG_CACHE_HOME', DEFAULT_XDG_CACHE_HOME)
snake_case__     							= os.path.join(XDG_CACHE_HOME, 'huggingface')
snake_case__     							= os.path.expanduser(os.getenv('HF_HOME', DEFAULT_HF_CACHE_HOME))

snake_case__     							= os.path.join(HF_CACHE_HOME, 'datasets')
snake_case__     							= Path(os.getenv('HF_DATASETS_CACHE', DEFAULT_HF_DATASETS_CACHE))

snake_case__     							= os.path.join(HF_CACHE_HOME, 'metrics')
snake_case__     							= Path(os.getenv('HF_METRICS_CACHE', DEFAULT_HF_METRICS_CACHE))

snake_case__     							= os.path.join(HF_CACHE_HOME, 'modules')
snake_case__     							= Path(os.getenv('HF_MODULES_CACHE', DEFAULT_HF_MODULES_CACHE))

snake_case__     							= 'downloads'
snake_case__     							= os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
snake_case__     							= Path(os.getenv('HF_DATASETS_DOWNLOADED_DATASETS_PATH', DEFAULT_DOWNLOADED_DATASETS_PATH))

snake_case__     							= 'extracted'
snake_case__     							= os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
snake_case__     							= Path(os.getenv('HF_DATASETS_EXTRACTED_DATASETS_PATH', DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
snake_case__     							= (
    os.environ.get('HF_UPDATE_DOWNLOAD_COUNTS', 'AUTO').upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
snake_case__     							= 1000

# Size of the preloaded record batch in `Dataset.__iter__`
snake_case__     							= 10

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
snake_case__     							= 4 << 30

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
snake_case__     							= '500MB'

# Parquet configuration
snake_case__     							= 100
snake_case__     							= 100
snake_case__     							= 100

# Offline mode
snake_case__     							= os.environ.get('HF_DATASETS_OFFLINE', 'AUTO').upper() in ENV_VARS_TRUE_VALUES

# In-memory
snake_case__     							= 0  # Disabled
snake_case__     							= float(os.environ.get('HF_DATASETS_IN_MEMORY_MAX_SIZE', DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
snake_case__     							= 'dataset.arrow'
snake_case__     							= 'indices.arrow'
snake_case__     							= 'state.json'
snake_case__     							= 'dataset_info.json'
snake_case__     							= 'dataset_infos.json'
snake_case__     							= 'LICENSE'
snake_case__     							= 'metric_info.json'
snake_case__     							= 'dataset_dict.json'

snake_case__     							= 'datasets_modules'

snake_case__     							= 255

# Streaming
snake_case__     							= 20
snake_case__     							= 5

# Datasets without script
snake_case__     							= 200
snake_case__     							= 10
snake_case__     							= 200

# Progress bars
snake_case__     							= 0.05  # 20 progress updates per sec






import collections

from .utils import ExplicitEnum, is_torch_available, logging


if is_torch_available():
			import torch


snake_case__     							= logging.get_logger(__name__)



class     snake_case__  :
				def __init__(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase=21     ,      _UpperCAmelCase=[]     ,      _UpperCAmelCase=None      ):




										"""simple docstring"""




										__magic_name__				:							Tuple				=      model
										__magic_name__				:							List[str]				=      trace_batch_nums
										__magic_name__				:							int				=      abort_after_batch_num

										# keep a LIFO buffer of frames to dump as soon as inf/nan is encountered to give context to the problem emergence
										__magic_name__				:							Union[str, Any]				=      collections.deque([]     ,      _UpperCAmelCase      )
										__magic_name__				:							Union[str, Any]				=      []
										__magic_name__				:							List[str]				=      0
										__magic_name__				:							Optional[int]				=      0
										__magic_name__				:							Any				=      False
										__magic_name__				:							str				=      '                 '

										self.analyse_model()

										self.register_forward_hook()
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase=None      ):




										"""simple docstring"""




										if frame is not None:
																self.expand_frame(_UpperCAmelCase      )
										self.frames.append('\n'.join(self.frame      )      )
										__magic_name__				:							List[Any]				=      []  # start a new frame
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										self.frame.append(_UpperCAmelCase      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										print('\n'.join(self.frames      )      )
										__magic_name__				:							List[str]				=      []
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							List[Any]				=      []
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										print(F"\nDetected inf/nan during batch_number={self.batch_number}"      )
										print(F"Last {len(self.frames      )} forward frames:"      )
										print(F"{'abs min':8} {'abs max':8} metadata"      )
										print('\n'.join(self.frames      )      )
										print('\n\n'      )
										__magic_name__				:							Any				=      []
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										__magic_name__				:							int				=      {m: name for name, m in self.model.named_modules()}
										# self.longest_module_name = max(len(v) for v in self.module_names.values())
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										if torch.is_tensor(_UpperCAmelCase      ):
																self.expand_frame(get_abs_min_max(_UpperCAmelCase     ,      _UpperCAmelCase      )      )
																if detect_overflow(_UpperCAmelCase     ,      _UpperCAmelCase      ):
																						__magic_name__				:							List[str]				=      True
										elif var is None:
																self.expand_frame(F"{'None':>17} {ctx}"      )
										else:
																self.expand_frame(F"{'not a tensor':>17} {ctx}"      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										self.expand_frame(F"\n\n{self.prefix} *** Starting batch number={self.batch_number} ***"      )
										self.expand_frame(F"{'abs min':8} {'abs max':8} metadata"      )
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										self.expand_frame(F"{self.prefix} *** Finished batch number={self.batch_number-1} ***\n\n"      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										self.expand_frame(F"{self.prefix} {self.module_names[module]} {module.__class__.__name__}"      )

										# params
										for name, p in module.named_parameters(recurse=_UpperCAmelCase      ):
																self.analyse_variable(_UpperCAmelCase     ,      _UpperCAmelCase      )

										# inputs
										if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ):
																for i, x in enumerate(_UpperCAmelCase      ):
																						self.analyse_variable(_UpperCAmelCase     ,      F"input[{i}]"      )
										else:
																self.analyse_variable(_UpperCAmelCase     ,      'input'      )

										# outputs
										if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ):
																for i, x in enumerate(_UpperCAmelCase      ):
																						# possibly a tuple of tuples
																						if isinstance(_UpperCAmelCase     ,      _UpperCAmelCase      ):
																												for j, y in enumerate(_UpperCAmelCase      ):
																																		self.analyse_variable(_UpperCAmelCase     ,      F"output[{i}][{j}]"      )
																						else:
																												self.analyse_variable(_UpperCAmelCase     ,      F"output[{i}]"      )
										else:
																self.analyse_variable(_UpperCAmelCase     ,      'output'      )

										self.save_frame()
				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										self.model.apply(self._register_forward_hook      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										module.register_forward_hook(self.forward_hook      )





				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										__magic_name__				:							Optional[int]				=      False

										__magic_name__				:							Any				=      True if self.batch_number in self.trace_batch_nums else False
										if trace_mode:
																self.reset_saved_frames()

										if self.total_calls == 0:
																self.batch_start_frame()
										self.total_calls += 1

										# count batch numbers - the very first forward hook of the batch will be called when the
										# batch completes - i.e. it gets called very last - we know this batch has finished
										if module == self.model:
																self.batch_number += 1
																__magic_name__				:							str				=      True

										self.create_frame(_UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase      )

										# if last_frame_of_batch:
										#     self.batch_end_frame()

										if trace_mode:
																self.trace_frames()

										if last_frame_of_batch:
																self.batch_start_frame()

										if self.detected_overflow and not trace_mode:
																self.dump_saved_frames()

																# now we can abort, as it's pointless to continue running
																raise ValueError(
																    'DebugUnderflowOverflow: inf/nan detected, aborting as there is no point running further. '
																    'Please scroll up above this traceback to see the activation values prior to this event.'      )

										# abort after certain batch if requested to do so
										if self.abort_after_batch_num is not None and self.batch_number > self.abort_after_batch_num:
																raise ValueError(
																    F"DebugUnderflowOverflow: aborting after {self.batch_number} batches due to"
																    F" `abort_after_batch_num={self.abort_after_batch_num}` arg"      )






def  __SCREAMING_SNAKE_CASE       (				lowercase__  :Union[str, Any]      ,						lowercase__  :Any				)       ->  Any:
						__magic_name__				:							List[Any]				=      var.abs()
						return f"{abs_var.min():8.2e} {abs_var.max():8.2e} {ctx}"
def  __SCREAMING_SNAKE_CASE       (				lowercase__  :Tuple      ,						lowercase__  :Optional[int]				)       ->  Optional[Any]:
						__magic_name__				:							str				=      False
						if torch.isnan(lowercase__				).any().item():
												__magic_name__				:							Optional[Any]				=      True
												print(f"{ctx} has nans"				)
						if torch.isinf(lowercase__				).any().item():
												__magic_name__				:							str				=      True
												print(f"{ctx} has infs"				)

						# if needed to monitor large elements can enable the following
						if 0:  # and detected:
												__magic_name__				:							Any				=      var[torch.ge(var.abs()      ,						1_0_0				)]
												if naaa.numel() > 0:
																		print(f"{ctx}:  n100={naaa.numel()}"				)
												__magic_name__				:							Union[str, Any]				=      var[torch.ge(var.abs()      ,						1_0_0_0				)]
												if naaaa.numel() > 0:
																		print(f"{ctx}: n1000={naaaa.numel()}"				)
												__magic_name__				:							Any				=      var[torch.ge(var.abs()      ,						1_0_0_0_0				)]
												if naaaaa.numel() > 0:
																		print(f"{ctx}: n10000={naaaaa.numel()}"				)

						if 0:
												print(f"min={var.min():9.2e} max={var.max():9.2e}"				)

						if 0:
												print(f"min={var.min():9.2e} max={var.max():9.2e} var={var.var():9.2e} mean={var.mean():9.2e} ({ctx})"				)

						return detected



class     snake_case__  (      _A  ):
				__lowerCAmelCase		=      '''underflow_overflow'''
				__lowerCAmelCase		=      '''tpu_metrics_debug'''


