





"""simple docstring"""



import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






class 	__snake_case      (	lowerCamelCase_  ):





	"""simple docstring"""


	def __init__(					self      ,		__lowerCamelCase      ,		__lowerCamelCase = None      ,		__lowerCamelCase = None      ,		__lowerCamelCase = None      ,		__lowerCamelCase = False      ,		__lowerCamelCase = False      ,		__lowerCamelCase = None      ,		**__lowerCamelCase      ,		):



					'''simple docstring'''




					super().__init__(
					    __lowerCamelCase      ,		split=__lowerCamelCase      ,		features=__lowerCamelCase      ,		cache_dir=__lowerCamelCase      ,		keep_in_memory=__lowerCamelCase      ,		streaming=__lowerCamelCase      ,		num_proc=__lowerCamelCase      ,		**__lowerCamelCase      ,		)
					__A					:		str						=  path_or_paths if isinstance(__lowerCamelCase      ,		__lowerCamelCase						) else {self.split: path_or_paths}
					__A					:		Tuple						=  Csv(
					    cache_dir=__lowerCamelCase      ,		data_files=__lowerCamelCase      ,		features=__lowerCamelCase      ,		**__lowerCamelCase      ,		)







	def       UpperCamelCase__(					self						):



					'''simple docstring'''




					if self.streaming:
									__A					:		Optional[int]						=  self.builder.as_streaming_dataset(split=self.split						)
					# Build regular (map-style) dataset
					else:
									__A					:		int						=  None
									__A					:		int						=  None
									__A					:		str						=  None
									__A					:		List[Any]						=  None

									self.builder.download_and_prepare(
									    download_config=__lowerCamelCase      ,		download_mode=__lowerCamelCase      ,		verification_mode=__lowerCamelCase      ,		base_path=__lowerCamelCase      ,		num_proc=self.num_proc      ,		)
									__A					:		int						=  self.builder.as_dataset(
									    split=self.split      ,		verification_mode=__lowerCamelCase      ,		in_memory=self.keep_in_memory						)
					return dataset






class 	__snake_case      :





	"""simple docstring"""


	def __init__(					self      ,		__lowerCamelCase      ,		__lowerCamelCase      ,		__lowerCamelCase = None      ,		__lowerCamelCase = None      ,		**__lowerCamelCase      ,		):



					'''simple docstring'''




					if num_proc is not None and num_proc <= 0:
									raise ValueError(F"""num_proc {num_proc} must be an integer > 0."""						)

					__A					:		List[str]						=  dataset
					__A					:		str						=  path_or_buf
					__A					:		Union[str, Any]						=  batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
					__A					:		List[str]						=  num_proc
					__A					:		Dict						=  "utf-8"
					__A					:		Optional[int]						=  to_csv_kwargs

	def       UpperCamelCase__(					self						):



					'''simple docstring'''




					__A					:		List[str]						=  self.to_csv_kwargs.pop('''path_or_buf'''      ,		__lowerCamelCase						)
					__A					:		str						=  self.to_csv_kwargs.pop('''header'''      ,		__lowerCamelCase						)
					__A					:		Union[str, Any]						=  self.to_csv_kwargs.pop('''index'''      ,		__lowerCamelCase						)

					if isinstance(self.path_or_buf      ,		(str, bytes, os.PathLike)						):
									with open(self.path_or_buf      ,		'''wb+'''						) as buffer:
													__A					:		str						=  self._write(file_obj=__lowerCamelCase      ,		header=__lowerCamelCase      ,		index=__lowerCamelCase      ,		**self.to_csv_kwargs						)
					else:
									__A					:		int						=  self._write(file_obj=self.path_or_buf      ,		header=__lowerCamelCase      ,		index=__lowerCamelCase      ,		**self.to_csv_kwargs						)
					return written

	def       UpperCamelCase__(					self      ,		__lowerCamelCase						):



					'''simple docstring'''




					__A					:		str						=  args

					__A					:		Union[str, Any]						=  query_table(
					    table=self.dataset.data      ,		key=slice(__lowerCamelCase      ,		offset + self.batch_size						)      ,		indices=self.dataset._indices      ,		)
					__A					:		str						=  batch.to_pandas().to_csv(
					    path_or_buf=__lowerCamelCase      ,		header=header if (offset == 0) else False      ,		index=__lowerCamelCase      ,		**__lowerCamelCase						)
					return csv_str.encode(self.encoding						)







	def       UpperCamelCase__(					self      ,		__lowerCamelCase      ,		__lowerCamelCase      ,		__lowerCamelCase      ,		**__lowerCamelCase						):



					'''simple docstring'''




					__A					:		str						=  0

					if self.num_proc is None or self.num_proc == 1:
									for offset in logging.tqdm(
									    range(0      ,		len(self.dataset						)      ,		self.batch_size						)      ,		unit='''ba'''      ,		disable=not logging.is_progress_bar_enabled()      ,		desc='''Creating CSV from Arrow format'''      ,		):
													__A					:		List[str]						=  self._batch_csv((offset, header, index, to_csv_kwargs)						)
													written += file_obj.write(__lowerCamelCase						)

					else:
									__A					:		Optional[Any]						=  len(self.dataset						), self.batch_size
									with multiprocessing.Pool(self.num_proc						) as pool:
													for csv_str in logging.tqdm(
													    pool.imap(
													        self._batch_csv      ,		[(offset, header, index, to_csv_kwargs) for offset in range(0      ,		__lowerCamelCase      ,		__lowerCamelCase						)]      ,		)      ,		total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size      ,		unit='''ba'''      ,		disable=not logging.is_progress_bar_enabled()      ,		desc='''Creating CSV from Arrow format'''      ,		):
																	written += file_obj.write(__lowerCamelCase						)

					return written











"""simple docstring"""



import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_video_inputs


if is_torch_available():
       import torch

if is_vision_available():
       from PIL import Image

       from transformers import VideoMAEImageProcessor






class 	__snake_case      (	unittest.TestCase  ):





 """simple docstring"""


 def __init__(					self      ,		__lowerCamelCase      ,		__lowerCamelCase=7      ,		__lowerCamelCase=3      ,		__lowerCamelCase=10      ,		__lowerCamelCase=18      ,		__lowerCamelCase=30      ,		__lowerCamelCase=400      ,		__lowerCamelCase=True      ,		__lowerCamelCase=None      ,		__lowerCamelCase=True      ,		__lowerCamelCase=[0.5, 0.5, 0.5]      ,		__lowerCamelCase=[0.5, 0.5, 0.5]      ,		__lowerCamelCase=None      ,		):



     '''simple docstring'''




     __A					:		Optional[Any]						=  size if size is not None else {'''shortest_edge''': 18}
     __A					:		Optional[int]						=  crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}

     __A					:		Any						=  parent
     __A					:		Union[str, Any]						=  batch_size
     __A					:		Optional[int]						=  num_channels
     __A					:		int						=  num_frames
     __A					:		Optional[int]						=  image_size
     __A					:		Union[str, Any]						=  min_resolution
     __A					:		int						=  max_resolution
     __A					:		Tuple						=  do_resize
     __A					:		str						=  size
     __A					:		Any						=  do_normalize
     __A					:		Union[str, Any]						=  image_mean
     __A					:		List[str]						=  image_std
     __A					:		Union[str, Any]						=  crop_size







 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     return {
         "image_mean": self.image_mean,
         "image_std": self.image_std,
         "do_normalize": self.do_normalize,
         "do_resize": self.do_resize,
         "size": self.size,
         "crop_size": self.crop_size,
     }






@require_torch
@require_vision
class 	__snake_case      (	SCREAMING_SNAKE_CASE__							,							unittest.TestCase  ):





 """simple docstring"""


 _lowerCamelCase				 =  VideoMAEImageProcessor if is_vision_available() else None

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		List[str]						=  VideoMAEImageProcessingTester(self						)

 @property
 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     return self.image_processor_tester.prepare_image_processor_dict()

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Dict						=  self.image_processing_class(**self.image_processor_dict						)
     self.assertTrue(hasattr(__lowerCamelCase      ,		'''image_mean'''						)						)
     self.assertTrue(hasattr(__lowerCamelCase      ,		'''image_std'''						)						)
     self.assertTrue(hasattr(__lowerCamelCase      ,		'''do_normalize'''						)						)
     self.assertTrue(hasattr(__lowerCamelCase      ,		'''do_resize'''						)						)
     self.assertTrue(hasattr(__lowerCamelCase      ,		'''do_center_crop'''						)						)
     self.assertTrue(hasattr(__lowerCamelCase      ,		'''size'''						)						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Optional[int]						=  self.image_processing_class.from_dict(self.image_processor_dict						)
     self.assertEqual(image_processor.size      ,		{'''shortest_edge''': 18}						)
     self.assertEqual(image_processor.crop_size      ,		{'''height''': 18, '''width''': 18}						)

     __A					:		Dict						=  self.image_processing_class.from_dict(self.image_processor_dict      ,		size=42      ,		crop_size=84						)
     self.assertEqual(image_processor.size      ,		{'''shortest_edge''': 42}						)
     self.assertEqual(image_processor.crop_size      ,		{'''height''': 84, '''width''': 84}						)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     pass

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		List[Any]						=  self.image_processing_class(**self.image_processor_dict						)
     # create random PIL videos
     __A					:		Optional[int]						=  prepare_video_inputs(self.image_processor_tester      ,		equal_resolution=__lowerCamelCase						)
     for video in video_inputs:
         self.assertIsInstance(__lowerCamelCase      ,		__lowerCamelCase						)
         self.assertIsInstance(video[0]      ,		Image.Image						)

     # Test not batched input
     __A					:		Any						=  image_processing(video_inputs[0]      ,		return_tensors='''pt'''						).pixel_values
     self.assertEqual(
         encoded_videos.shape      ,		(
             1,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         )      ,		)

     # Test batched
     __A					:		int						=  image_processing(__lowerCamelCase      ,		return_tensors='''pt'''						).pixel_values
     self.assertEqual(
         encoded_videos.shape      ,		(
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         )      ,		)

 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		List[Any]						=  self.image_processing_class(**self.image_processor_dict						)
     # create random numpy tensors
     __A					:		List[str]						=  prepare_video_inputs(self.image_processor_tester      ,		equal_resolution=__lowerCamelCase      ,		numpify=__lowerCamelCase						)
     for video in video_inputs:
         self.assertIsInstance(__lowerCamelCase      ,		__lowerCamelCase						)
         self.assertIsInstance(video[0]      ,		np.ndarray						)

     # Test not batched input
     __A					:		List[str]						=  image_processing(video_inputs[0]      ,		return_tensors='''pt'''						).pixel_values
     self.assertEqual(
         encoded_videos.shape      ,		(
             1,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         )      ,		)

     # Test batched
     __A					:		int						=  image_processing(__lowerCamelCase      ,		return_tensors='''pt'''						).pixel_values
     self.assertEqual(
         encoded_videos.shape      ,		(
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         )      ,		)







 def       UpperCamelCase__(					self						):



     '''simple docstring'''




     __A					:		Optional[int]						=  self.image_processing_class(**self.image_processor_dict						)
     # create random PyTorch tensors
     __A					:		int						=  prepare_video_inputs(self.image_processor_tester      ,		equal_resolution=__lowerCamelCase      ,		torchify=__lowerCamelCase						)
     for video in video_inputs:
         self.assertIsInstance(__lowerCamelCase      ,		__lowerCamelCase						)
         self.assertIsInstance(video[0]      ,		torch.Tensor						)

     # Test not batched input
     __A					:		Optional[int]						=  image_processing(video_inputs[0]      ,		return_tensors='''pt'''						).pixel_values
     self.assertEqual(
         encoded_videos.shape      ,		(
             1,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         )      ,		)

     # Test batched
     __A					:		str						=  image_processing(__lowerCamelCase      ,		return_tensors='''pt'''						).pixel_values
     self.assertEqual(
         encoded_videos.shape      ,		(
             self.image_processor_tester.batch_size,
             self.image_processor_tester.num_frames,
             self.image_processor_tester.num_channels,
             self.image_processor_tester.crop_size['''height'''],
             self.image_processor_tester.crop_size['''width'''],
         )      ,		)




