


import torch

from accelerate import Accelerator, DistributedType



class    __lowercase  :

							"""simple docstring"""


							def __enter__(    self	)->	Any:
											if self.enabled:
															_SCREAMING_SNAKE_CASE      = self.model.no_sync()
															self.model_sync_obj.__enter__()

											return self





							def __exit__(    self				, A_				, A_				, A_	)->	Any:
											if self.enabled:
															# Average all models on exit
															self._sync_and_avg_model_params()
															self.model_sync_obj.__exit__(__UpperCamelCase				, __UpperCamelCase				, __UpperCamelCase	)





							def __init__(    self				, A_				, A_				, A_				, A_ = True	)->	str:
											if accelerator.distributed_type not in [
											    DistributedType.NO,
											    DistributedType.MULTI_CPU,
											    DistributedType.MULTI_GPU,
											]:
															raise NotImplementedError('LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'	)
											_SCREAMING_SNAKE_CASE      = enabled and accelerator.distributed_type != DistributedType.NO
											_SCREAMING_SNAKE_CASE      = 0
											if self.enabled:
															_SCREAMING_SNAKE_CASE      = accelerator
															_SCREAMING_SNAKE_CASE      = model
															_SCREAMING_SNAKE_CASE      = local_sgd_steps





							def 		__magic_name__			(    self	)->	Optional[int]:
											self.num_steps += 1
											if not self.enabled:
															return

											if self.num_steps % self.local_sgd_steps == 0:
															self._sync_and_avg_model_params()





							def 		__magic_name__			(    self	)->	Optional[Any]:
											self.accelerator.wait_for_everyone()
											with self.accelerator.autocast():
															for param in self.model.parameters():
																			_SCREAMING_SNAKE_CASE      = self.accelerator.reduce(param.data				, reduction='mean'	)




import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
							import sqlitea

							import sqlalchemy


snake_case			:      List[str] 							=   datasets.utils.logging.get_logger(__name__)



@dataclass
class    __lowercase  (       datasets.BuilderConfig							):

							"""simple docstring"""


							SCREAMING_SNAKE_CASE   :					Union[str, "sqlalchemy.sql.Selectable"]        =						None
							SCREAMING_SNAKE_CASE   :					Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]        =						None
							SCREAMING_SNAKE_CASE   :					Optional[Union[str, List[str]]]        =						None
							SCREAMING_SNAKE_CASE   :					bool        =						True
							SCREAMING_SNAKE_CASE   :					Optional[Union[List, Tuple, Dict]]        =						None
							SCREAMING_SNAKE_CASE   :					Optional[Union[List, Dict]]        =						None
							SCREAMING_SNAKE_CASE   :					Optional[List[str]]        =						None
							SCREAMING_SNAKE_CASE   :					Optional[int]        =						1_00_00
							SCREAMING_SNAKE_CASE   :					Optional[datasets.Features]        =						None





							def 		__magic_name__			(    self	)->	int:
											if self.sql is None:
															raise ValueError('sql must be specified'	)
											if self.con is None:
															raise ValueError('con must be specified'	)





							def 		__magic_name__			(    self				, A_				, A_ = None				, )->	str:
											_SCREAMING_SNAKE_CASE      = config_kwargs.copy()
											# We need to stringify the Selectable object to make its hash deterministic

											# The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
											_SCREAMING_SNAKE_CASE      = config_kwargs['sql']
											if not isinstance(A_				, A_	):
															if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
																			import sqlalchemy

																			if isinstance(A_				, sqlalchemy.sql.Selectable	):
																							_SCREAMING_SNAKE_CASE      = sqlalchemy.create_engine(config_kwargs['con'].split('://'	)[0] + '://'	)
																							_SCREAMING_SNAKE_CASE      = str(sql.compile(dialect=engine.dialect	)	)
																							_SCREAMING_SNAKE_CASE      = sql_str
																			else:
																							raise TypeError(
																							    F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(A_	)}: {sql}'''	)
															else:
																			raise TypeError(
																			    F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(A_	)}: {sql}'''	)
											_SCREAMING_SNAKE_CASE      = config_kwargs['con']
											if not isinstance(A_				, A_	):
															_SCREAMING_SNAKE_CASE      = id(A_	)
															logger.info(
															    F'''SQL connection \'con\' of type {type(A_	)} couldn\'t be hashed properly. To enable hashing, specify \'con\' as URI string instead.'''	)

											return super().create_config_id(A_				, custom_features=A_	)





							@property
							def 		__magic_name__			(    self	)->	Dict:
											_SCREAMING_SNAKE_CASE      = {
											    'index_col': self.index_col,
											    'columns': self.columns,
											    'params': self.params,
											    'coerce_float': self.coerce_float,
											    'parse_dates': self.parse_dates,
											}
											return pd_read_sql_kwargs







class    __lowercase  (       datasets.ArrowBasedBuilder							):

							"""simple docstring"""


							SCREAMING_SNAKE_CASE   :					Any        =						SqlConfig





							def 		__magic_name__			(    self	)->	Dict:
											return datasets.DatasetInfo(features=self.config.features	)





							def 		__magic_name__			(    self				, A_	)->	Optional[Any]:
											return [datasets.SplitGenerator(name=datasets.Split.TRAIN				, gen_kwargs={}	)]





							def 		__magic_name__			(    self				, A_	)->	pa.Table:
											if self.config.features is not None:
															_SCREAMING_SNAKE_CASE      = self.config.features.arrow_schema
															if all(not require_storage_cast(A_	) for feature in self.config.features.values()	):
																			# cheaper cast
																			_SCREAMING_SNAKE_CASE      = pa.Table.from_arrays([pa_table[field.name] for field in schema]				, schema=A_	)
															else:
																			# more expensive cast; allows str <-> int/float or str to Audio for example
																			_SCREAMING_SNAKE_CASE      = table_cast(A_				, A_	)
											return pa_table





							def 		__magic_name__			(    self	)->	Optional[int]:
											_SCREAMING_SNAKE_CASE      = self.config.chunksize
											_SCREAMING_SNAKE_CASE      = pd.read_sql(
											    self.config.sql				, self.config.con				, chunksize=A_				, **self.config.pd_read_sql_kwargs	)
											_SCREAMING_SNAKE_CASE      = [sql_reader] if chunksize is None else sql_reader
											for chunk_idx, df in enumerate(A_	):
															_SCREAMING_SNAKE_CASE      = pa.Table.from_pandas(A_	)
															yield chunk_idx, self._cast_table(A_	)
