

from dataclasses import dataclass
from typing import List, Optional, Union

import numpy as np
import PIL

from ...utils import BaseOutput, is_invisible_watermark_available, is_torch_available, is_transformers_available



@dataclass
class 						__lowercase       (				__lowerCamelCase			):




   """simple docstring"""







   _UpperCAmelCase     :							Union[List[PIL.Image.Image], np.ndarray]


if is_transformers_available() and is_torch_available() and is_invisible_watermark_available():
      from .pipeline_stable_diffusion_xl import StableDiffusionXLPipeline
      from .pipeline_stable_diffusion_xl_imgaimg import StableDiffusionXLImgaImgPipeline









import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


lowerCAmelCase							:						Tuple     			=		datasets.utils.logging.get_logger(__name__)



@dataclass
class 						__lowercase       (				datasets.BuilderConfig			):




			"""simple docstring"""







			_UpperCAmelCase     :							Optional[datasets.Features]    							=  None
			_UpperCAmelCase     :							str    							=  "utf-8"
			_UpperCAmelCase     :							Optional[str]    							=  None
			_UpperCAmelCase     :							int    							=  10 << 20  # 10MB
			_UpperCAmelCase     :							bool    							=  False
			_UpperCAmelCase     :							str    							=  "line"







class 						__lowercase       (				datasets.ArrowBasedBuilder			):




			"""simple docstring"""







			_UpperCAmelCase     :							List[Any]    							=  TextConfig



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[Any]):
									return datasets.DatasetInfo(features=self.config.features)



			def  _SCREAMING_SNAKE_CASE    (				self						:  List[Any]     ,		lowerCAmelCase__						:  Tuple):
									if not self.config.data_files:
															raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}")
									SCREAMING_SNAKE_CASE_:				Dict							= dl_manager.download_and_extract(self.config.data_files)
									if isinstance(lowerCAmelCase__     ,		(str, list, tuple)):
															SCREAMING_SNAKE_CASE_:				Dict							= data_files
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				Dict							= [files]
															SCREAMING_SNAKE_CASE_:				List[str]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,		gen_kwargs={"files": files})]
									SCREAMING_SNAKE_CASE_:				Any							= []
									for split_name, files in data_files.items():
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				List[Any]							= [files]
															SCREAMING_SNAKE_CASE_:				Union[str, Any]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															splits.append(datasets.SplitGenerator(name=lowerCAmelCase__     ,		gen_kwargs={"files": files}))
									return splits



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[int]     ,		lowerCAmelCase__						:  pa.Table):
									if self.config.features is not None:
															SCREAMING_SNAKE_CASE_:				Optional[int]							= self.config.features.arrow_schema
															if all(not require_storage_cast(lowerCAmelCase__) for feature in self.config.features.values()):
																					# cheaper cast
																					SCREAMING_SNAKE_CASE_:				List[str]							= pa_table.cast(lowerCAmelCase__)
															else:
																					# more expensive cast; allows str <-> int/float or str to Audio for example
																					SCREAMING_SNAKE_CASE_:				List[Any]							= table_cast(lowerCAmelCase__     ,		lowerCAmelCase__)
															return pa_table
									else:
															return pa_table.cast(pa.schema({"text": pa.string()}))




			def  _SCREAMING_SNAKE_CASE    (				self						:  int     ,		lowerCAmelCase__						:  Dict):
									SCREAMING_SNAKE_CASE_:				str							= list(self.config.features) if self.config.features is not None else ["text"]
									for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase__)):
															# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
															with open(lowerCAmelCase__     ,		encoding=self.config.encoding     ,		errors=self.config.errors) as f:
																					if self.config.sample_by == "line":
																											SCREAMING_SNAKE_CASE_:				Any							= 0
																											while True:
																																	SCREAMING_SNAKE_CASE_:				str							= f.read(self.config.chunksize)
																																	if not batch:
																																							break
																																	batch += f.readline()  # finish current line
																																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= StringIO(lowerCAmelCase__).readlines()
																																	if not self.config.keep_linebreaks:
																																							SCREAMING_SNAKE_CASE_:				Optional[Any]							= [line.rstrip("\n") for line in batch]
																																	SCREAMING_SNAKE_CASE_:				Any							= pa.Table.from_arrays([pa.array(lowerCAmelCase__)]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																					elif self.config.sample_by == "paragraph":
																											SCREAMING_SNAKE_CASE_:				List[str]							= 0
																											SCREAMING_SNAKE_CASE_:				Tuple							= ""
																											while True:
																																	SCREAMING_SNAKE_CASE_:				Dict							= f.read(self.config.chunksize)
																																	if not new_batch:
																																							break
																																	batch += new_batch
																																	batch += f.readline()  # finish current line
																																	SCREAMING_SNAKE_CASE_:				List[str]							= batch.split("\n\n")
																																	SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays(
																																	    [pa.array([example for example in batch[:-1] if example])]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= batch[-1]
																											if batch:
																																	SCREAMING_SNAKE_CASE_:				int							= pa.Table.from_arrays([pa.array([batch])]     ,		names=lowerCAmelCase__)
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																					elif self.config.sample_by == "document":
																											SCREAMING_SNAKE_CASE_:				List[Any]							= f.read()
																											SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays([pa.array([text])]     ,		names=lowerCAmelCase__)
																											yield file_idx, self._cast_table(lowerCAmelCase__)






