
'''simple docstring'''







import argparse
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_dummies.py
_lowercase				:						Any												= "src/transformers"

# Matches is_xxx_available()
_lowercase				:						Tuple												= re.compile(r"is\_([a-z_]*)_available()")
# Matches from xxx import bla
_lowercase				:						Dict												= re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")
_lowercase				:						int												= re.compile(r"^\s+if\s+not\s+\(?is\_[a-z_]*\_available\(\)")


_lowercase				:						List[str]												= "\n{0} = None\n"

_lowercase				:						Tuple												= "\nclass {0}(metaclass=DummyObject):\n    _backends = {1}\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, {1})\n"


_lowercase				:						Union[str, Any]												= "\ndef {0}(*args, **kwargs):\n    requires_backends({0}, {1})\n"
def       snake_case_	(       __SCREAMING_SNAKE_CASE      :				Tuple							):



						"""simple docstring"""





						if _re_test_backend.search(__SCREAMING_SNAKE_CASE							) is None:
												return None
						lowercase_			:  Any									= [b[0] for b in _re_backend.findall(__SCREAMING_SNAKE_CASE							)]
						backends.sort()
						return "_and_".join(__SCREAMING_SNAKE_CASE							)
def       snake_case_	(       ):



						"""simple docstring"""





						with open(os.path.join(__SCREAMING_SNAKE_CASE   ,   '''__init__.py'''							)   ,   '''r'''   ,   encoding='''utf-8'''   ,   newline='''\n'''							) as f:
												lowercase_			:  Optional[int]									= f.readlines()

						# Get to the point we do the actual imports for type checking
						lowercase_			:  Union[str, Any]									= 0
						while not lines[line_index].startswith('''if TYPE_CHECKING'''							):
												line_index += 1

						lowercase_			:  Optional[Any]									= {}
						# Go through the end of the file
						while line_index < len(__SCREAMING_SNAKE_CASE							):
												# If the line is an if is_backend_available, we grab all objects associated.
												lowercase_			:  Optional[int]									= find_backend(lines[line_index]							)
												if backend is not None:
																		while not lines[line_index].startswith('''    else:'''							):
																								line_index += 1
																		line_index += 1

																		lowercase_			:  List[Any]									= []
																		# Until we unindent, add backend objects to the list
																		while len(lines[line_index]							) <= 1 or lines[line_index].startswith(''' ''' * 8							):
																								lowercase_			:  int									= lines[line_index]
																								lowercase_			:  int									= _re_single_line_import.search(__SCREAMING_SNAKE_CASE							)
																								if single_line_import_search is not None:
																														objects.extend(single_line_import_search.groups()[0].split(''', '''							)							)
																								elif line.startswith(''' ''' * 12							):
																														objects.append(line[12:-2]							)
																								line_index += 1

																		lowercase_			:  int									= objects
												else:
																		line_index += 1

						return backend_specific_objects
def       snake_case_	(       __SCREAMING_SNAKE_CASE      :				List[str]   ,   __SCREAMING_SNAKE_CASE      :				int							):



						"""simple docstring"""





						if name.isupper():
												return DUMMY_CONSTANT.format(__SCREAMING_SNAKE_CASE							)
						elif name.islower():
												return DUMMY_FUNCTION.format(__SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE							)
						else:
												return DUMMY_CLASS.format(__SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE							)
def       snake_case_	(       __SCREAMING_SNAKE_CASE      :				Any=None							):



						"""simple docstring"""





						if backend_specific_objects is None:
												lowercase_			:  Dict									= read_init()
						# For special correspondence backend to module name as used in the function requires_modulename
						lowercase_			:  Optional[Any]									= {}

						for backend, objects in backend_specific_objects.items():
												lowercase_			:  Optional[Any]									= '''[''' + ''', '''.join(F'''"{b}"''' for b in backend.split('''_and_'''							)							) + ''']'''
												lowercase_			:  List[str]									= '''# This file is autogenerated by the command `make fix-copies`, do not edit.\n'''
												dummy_file += "from ..utils import DummyObject, requires_backends\n\n"
												dummy_file += "\n".join([create_dummy_object(__SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE							) for o in objects]							)
												lowercase_			:  List[Any]									= dummy_file

						return dummy_files
def       snake_case_	(       __SCREAMING_SNAKE_CASE      :				Optional[Any]=False							):



						"""simple docstring"""





						lowercase_			:  Optional[Any]									= create_dummy_files()
						# For special correspondence backend to shortcut as used in utils/dummy_xxx_objects.py
						lowercase_			:  Optional[int]									= {'''torch''': '''pt'''}

						# Locate actual dummy modules and read their content.
						lowercase_			:  Union[str, Any]									= os.path.join(__SCREAMING_SNAKE_CASE   ,   '''utils'''							)
						lowercase_			:  List[Any]									= {
						    backend: os.path.join(__SCREAMING_SNAKE_CASE   ,   F'''dummy_{short_names.get(__SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE							)}_objects.py'''							)
						    for backend in dummy_files.keys()
						}

						lowercase_			:  int									= {}
						for backend, file_path in dummy_file_paths.items():
												if os.path.isfile(__SCREAMING_SNAKE_CASE							):
																		with open(__SCREAMING_SNAKE_CASE   ,   '''r'''   ,   encoding='''utf-8'''   ,   newline='''\n'''							) as f:
																								lowercase_			:  str									= f.read()
												else:
																		lowercase_			:  Optional[Any]									= ''''''

						for backend in dummy_files.keys():
												if dummy_files[backend] != actual_dummies[backend]:
																		if overwrite:
																								print(
																								    F'''Updating transformers.utils.dummy_{short_names.get(__SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE							)}_objects.py as the main '''
																								    '''__init__ has new objects.'''							)
																								with open(dummy_file_paths[backend]   ,   '''w'''   ,   encoding='''utf-8'''   ,   newline='''\n'''							) as f:
																														f.write(dummy_files[backend]							)
																		else:
																								raise ValueError(
																								    '''The main __init__ has objects that are not present in '''
																								    F'''transformers.utils.dummy_{short_names.get(__SCREAMING_SNAKE_CASE   ,   __SCREAMING_SNAKE_CASE							)}_objects.py. Run `make fix-copies` '''
																								    '''to fix this.'''							)


if __name__ == "__main__":
							_lowercase				:						str												= argparse.ArgumentParser()
							parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
							_lowercase				:						Any												= parser.parse_args()

							check_dummies(args.fix_and_overwrite)





'''simple docstring'''







import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






class 				lowerCAmelCase__     (						lowerCamelCase_  ):






					def __init__(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = False       ,							__SCREAMING_SNAKE_CASE = False       ,							__SCREAMING_SNAKE_CASE = None       ,							**__SCREAMING_SNAKE_CASE       ,							):


											"""simple docstring"""





											super().__init__(
											    __SCREAMING_SNAKE_CASE       ,							split=__SCREAMING_SNAKE_CASE       ,							features=__SCREAMING_SNAKE_CASE       ,							cache_dir=__SCREAMING_SNAKE_CASE       ,							keep_in_memory=__SCREAMING_SNAKE_CASE       ,							streaming=__SCREAMING_SNAKE_CASE       ,							num_proc=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE       ,							)
											lowercase_			:  int									= path_or_paths if isinstance(__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	) else {self.split: path_or_paths}
											lowercase_			:  List[Any]									= Csv(
											    cache_dir=__SCREAMING_SNAKE_CASE       ,							data_files=__SCREAMING_SNAKE_CASE       ,							features=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE       ,							)






					def 			_snake_case					(     self	):


											"""simple docstring"""





											if self.streaming:
																	lowercase_			:  List[str]									= self.builder.as_streaming_dataset(split=self.split	)
											# Build regular (map-style) dataset
											else:
																	lowercase_			:  Any									= None
																	lowercase_			:  Optional[Any]									= None
																	lowercase_			:  str									= None
																	lowercase_			:  Optional[int]									= None

																	self.builder.download_and_prepare(
																	    download_config=__SCREAMING_SNAKE_CASE       ,							download_mode=__SCREAMING_SNAKE_CASE       ,							verification_mode=__SCREAMING_SNAKE_CASE       ,							base_path=__SCREAMING_SNAKE_CASE       ,							num_proc=self.num_proc       ,							)
																	lowercase_			:  Tuple									= self.builder.as_dataset(
																	    split=self.split       ,							verification_mode=__SCREAMING_SNAKE_CASE       ,							in_memory=self.keep_in_memory	)
											return dataset







class 				lowerCAmelCase__     :






					def __init__(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE = None       ,							__SCREAMING_SNAKE_CASE = None       ,							**__SCREAMING_SNAKE_CASE       ,							):


											"""simple docstring"""





											if num_proc is not None and num_proc <= 0:
																	raise ValueError(F'''num_proc {num_proc} must be an integer > 0.'''	)

											lowercase_			:  List[Any]									= dataset
											lowercase_			:  Dict									= path_or_buf
											lowercase_			:  int									= batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
											lowercase_			:  int									= num_proc
											lowercase_			:  Optional[int]									= '''utf-8'''
											lowercase_			:  Optional[Any]									= to_csv_kwargs






					def 			_snake_case					(     self	):


											"""simple docstring"""





											lowercase_			:  List[str]									= self.to_csv_kwargs.pop('''path_or_buf'''       ,							__SCREAMING_SNAKE_CASE	)
											lowercase_			:  Optional[Any]									= self.to_csv_kwargs.pop('''header'''       ,							__SCREAMING_SNAKE_CASE	)
											lowercase_			:  Dict									= self.to_csv_kwargs.pop('''index'''       ,							__SCREAMING_SNAKE_CASE	)

											if isinstance(self.path_or_buf       ,							(str, bytes, os.PathLike)	):
																	with open(self.path_or_buf       ,							'''wb+'''	) as buffer:
																							lowercase_			:  Any									= self._write(file_obj=__SCREAMING_SNAKE_CASE       ,							header=__SCREAMING_SNAKE_CASE       ,							index=__SCREAMING_SNAKE_CASE       ,							**self.to_csv_kwargs	)
											else:
																	lowercase_			:  str									= self._write(file_obj=self.path_or_buf       ,							header=__SCREAMING_SNAKE_CASE       ,							index=__SCREAMING_SNAKE_CASE       ,							**self.to_csv_kwargs	)
											return written






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_		,				lowercase_		,				lowercase_		,				lowercase_			:  int									= args

											lowercase_			:  List[str]									= query_table(
											    table=self.dataset.data       ,							key=slice(__SCREAMING_SNAKE_CASE       ,							offset + self.batch_size	)       ,							indices=self.dataset._indices       ,							)
											lowercase_			:  Dict									= batch.to_pandas().to_csv(
											    path_or_buf=__SCREAMING_SNAKE_CASE       ,							header=header if (offset == 0) else False       ,							index=__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	)
											return csv_str.encode(self.encoding	)






					def 			_snake_case					(     self       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE       ,							**__SCREAMING_SNAKE_CASE	):


											"""simple docstring"""





											lowercase_			:  Dict									= 0

											if self.num_proc is None or self.num_proc == 1:
																	for offset in logging.tqdm(
																	    range(0       ,							len(self.dataset	)       ,							self.batch_size	)       ,							unit='''ba'''       ,							disable=not logging.is_progress_bar_enabled()       ,							desc='''Creating CSV from Arrow format'''       ,							):
																							lowercase_			:  Any									= self._batch_csv((offset, header, index, to_csv_kwargs)	)
																							written += file_obj.write(__SCREAMING_SNAKE_CASE	)

											else:
																	lowercase_		,				lowercase_			:  Union[str, Any]									= len(self.dataset	), self.batch_size
																	with multiprocessing.Pool(self.num_proc	) as pool:
																							for csv_str in logging.tqdm(
																							    pool.imap(
																							        self._batch_csv       ,							[(offset, header, index, to_csv_kwargs) for offset in range(0       ,							__SCREAMING_SNAKE_CASE       ,							__SCREAMING_SNAKE_CASE	)]       ,							)       ,							total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size       ,							unit='''ba'''       ,							disable=not logging.is_progress_bar_enabled()       ,							desc='''Creating CSV from Arrow format'''       ,							):
																													written += file_obj.write(__SCREAMING_SNAKE_CASE	)

											return written



