

import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
      import torch

if is_vision_available():
      from PIL import Image

      from transformers import TvltImageProcessor







def     A_   (					_UpperCAmelCase       ,			_UpperCAmelCase=10       ,			_UpperCAmelCase=10       ,			_UpperCAmelCase=False       ,			_UpperCAmelCase=False					):
      SCREAMING_SNAKE_CASE_:				List[Any]							= []
      for i in range(image_processor_tester.num_frames					):
            video.append(np.random.randint(2_55       ,			size=(image_processor_tester.num_channels, width, height)       ,			dtype=np.uinta					)					)

      if not numpify and not torchify:
            # PIL expects the channel dimension as last dimension
            SCREAMING_SNAKE_CASE_:				Optional[Any]							= [Image.fromarray(np.moveaxis(_UpperCAmelCase       ,			0       ,			-1					)					) for frame in video]

      if torchify:
            SCREAMING_SNAKE_CASE_:				List[str]							= [torch.from_numpy(_UpperCAmelCase					) for frame in video]

      return video







def     A_   (					_UpperCAmelCase       ,			_UpperCAmelCase=False       ,			_UpperCAmelCase=False       ,			_UpperCAmelCase=False					):
      assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

      SCREAMING_SNAKE_CASE_:				str							= []
      for i in range(image_processor_tester.batch_size					):
            if equal_resolution:
                  SCREAMING_SNAKE_CASE_:				Any							= image_processor_tester.max_resolution
            else:
                  SCREAMING_SNAKE_CASE_:				Union[str, Any]							= np.random.choice(
                      np.arange(image_processor_tester.min_resolution       ,			image_processor_tester.max_resolution					)       ,			2					)
                  SCREAMING_SNAKE_CASE_:				List[Any]							= prepare_video(
                      image_processor_tester=_UpperCAmelCase       ,			width=_UpperCAmelCase       ,			height=_UpperCAmelCase       ,			numpify=_UpperCAmelCase       ,			torchify=_UpperCAmelCase       ,			)
            video_inputs.append(_UpperCAmelCase					)

      return video_inputs



class 						__lowercase       (				unittest.TestCase			):




   """simple docstring"""







   def __init__(				self						:  Optional[int]     ,		lowerCAmelCase__						:  Optional[int]     ,		lowerCAmelCase__						:  Any=7     ,		lowerCAmelCase__						:  Union[str, Any]=3     ,		lowerCAmelCase__						:  List[Any]=4     ,		lowerCAmelCase__						:  str=18     ,		lowerCAmelCase__						:  Tuple=30     ,		lowerCAmelCase__						:  int=400     ,		lowerCAmelCase__						:  Dict=True     ,		lowerCAmelCase__						:  Union[str, Any]=None     ,		lowerCAmelCase__						:  Any=True     ,		lowerCAmelCase__						:  Union[str, Any]=[0.5, 0.5, 0.5]     ,		lowerCAmelCase__						:  Optional[int]=[0.5, 0.5, 0.5]     ,		lowerCAmelCase__						:  Union[str, Any]=True     ,		lowerCAmelCase__						:  Optional[int]=None     ,		):
         SCREAMING_SNAKE_CASE_:				str							= size if size is not None else {"shortest_edge": 18}
         SCREAMING_SNAKE_CASE_:				int							= crop_size if crop_size is not None else {"height": 18, "width": 18}

         SCREAMING_SNAKE_CASE_:				Dict							= parent
         SCREAMING_SNAKE_CASE_:				str							= batch_size
         SCREAMING_SNAKE_CASE_:				Optional[Any]							= num_channels
         SCREAMING_SNAKE_CASE_:				str							= num_frames
         SCREAMING_SNAKE_CASE_:				Any							= image_size
         SCREAMING_SNAKE_CASE_:				Dict							= min_resolution
         SCREAMING_SNAKE_CASE_:				Any							= max_resolution
         SCREAMING_SNAKE_CASE_:				Optional[Any]							= do_resize
         SCREAMING_SNAKE_CASE_:				Dict							= size
         SCREAMING_SNAKE_CASE_:				int							= do_normalize
         SCREAMING_SNAKE_CASE_:				Any							= image_mean
         SCREAMING_SNAKE_CASE_:				List[str]							= image_std
         SCREAMING_SNAKE_CASE_:				Union[str, Any]							= do_center_crop
         SCREAMING_SNAKE_CASE_:				Optional[Any]							= crop_size




   def  _SCREAMING_SNAKE_CASE    (				self						:  List[str]):
         return {
             "image_mean": self.image_mean,
             "image_std": self.image_std,
             "do_normalize": self.do_normalize,
             "do_resize": self.do_resize,
             "size": self.size,
             "do_center_crop": self.do_center_crop,
             "crop_size": self.crop_size,
         }







@require_torch
@require_vision
class 						__lowercase       (				UpperCAmelCase_	, unittest.TestCase			):




   """simple docstring"""







   _UpperCAmelCase     :							Optional[int]    							=  TvltImageProcessor if is_vision_available() else None



   def  _SCREAMING_SNAKE_CASE    (				self						:  List[str]):
         SCREAMING_SNAKE_CASE_:				List[Any]							= TvltImageProcessorTester(self)



   @property
   def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[int]):
         return self.image_processor_tester.prepare_image_processor_dict()



   def  _SCREAMING_SNAKE_CASE    (				self						:  str):
         SCREAMING_SNAKE_CASE_:				Any							= self.image_processing_class(**self.image_processor_dict)
         self.assertTrue(hasattr(lowerCAmelCase__     ,		"image_mean"))
         self.assertTrue(hasattr(lowerCAmelCase__     ,		"image_std"))
         self.assertTrue(hasattr(lowerCAmelCase__     ,		"do_normalize"))
         self.assertTrue(hasattr(lowerCAmelCase__     ,		"do_resize"))
         self.assertTrue(hasattr(lowerCAmelCase__     ,		"do_center_crop"))
         self.assertTrue(hasattr(lowerCAmelCase__     ,		"size"))



   def  _SCREAMING_SNAKE_CASE    (				self						:  Tuple):
         # Initialize image_processor
         SCREAMING_SNAKE_CASE_:				Optional[Any]							= self.image_processing_class(**self.image_processor_dict)
         # create random PIL videos
         SCREAMING_SNAKE_CASE_:				Tuple							= prepare_video_inputs(self.image_processor_tester     ,		equal_resolution=lowerCAmelCase__)
         for video in video_inputs:
               self.assertIsInstance(lowerCAmelCase__     ,		lowerCAmelCase__)
               self.assertIsInstance(video[0]     ,		Image.Image)

         # Test not batched input
         SCREAMING_SNAKE_CASE_:				str							= image_processor(video_inputs[0]     ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_videos.shape     ,		(
                 1,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size["height"],
                 self.image_processor_tester.crop_size["width"],
             )     ,		)

         # Test batched
         SCREAMING_SNAKE_CASE_:				int							= image_processor(lowerCAmelCase__     ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_videos.shape     ,		(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size["height"],
                 self.image_processor_tester.crop_size["width"],
             )     ,		)



   def  _SCREAMING_SNAKE_CASE    (				self						:  Dict):
         # Initialize image_processor
         SCREAMING_SNAKE_CASE_:				Dict							= self.image_processing_class(**self.image_processor_dict)
         # create random numpy tensors
         SCREAMING_SNAKE_CASE_:				Optional[Any]							= prepare_video_inputs(self.image_processor_tester     ,		equal_resolution=lowerCAmelCase__     ,		numpify=lowerCAmelCase__)
         for video in video_inputs:
               self.assertIsInstance(lowerCAmelCase__     ,		lowerCAmelCase__)
               self.assertIsInstance(video[0]     ,		np.ndarray)

         # Test not batched input
         SCREAMING_SNAKE_CASE_:				int							= image_processor(video_inputs[0]     ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_videos.shape     ,		(
                 1,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size["height"],
                 self.image_processor_tester.crop_size["width"],
             )     ,		)

         # Test batched
         SCREAMING_SNAKE_CASE_:				Any							= image_processor(lowerCAmelCase__     ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_videos.shape     ,		(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size["height"],
                 self.image_processor_tester.crop_size["width"],
             )     ,		)




   def  _SCREAMING_SNAKE_CASE    (				self						:  int):
         # Initialize image_processor
         SCREAMING_SNAKE_CASE_:				str							= self.image_processing_class(**self.image_processor_dict)
         # create random PyTorch tensors
         SCREAMING_SNAKE_CASE_:				List[Any]							= prepare_video_inputs(self.image_processor_tester     ,		equal_resolution=lowerCAmelCase__     ,		torchify=lowerCAmelCase__)
         for video in video_inputs:
               self.assertIsInstance(lowerCAmelCase__     ,		lowerCAmelCase__)
               self.assertIsInstance(video[0]     ,		torch.Tensor)

         # Test not batched input
         SCREAMING_SNAKE_CASE_:				Tuple							= image_processor(video_inputs[0]     ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_videos.shape     ,		(
                 1,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size["height"],
                 self.image_processor_tester.crop_size["width"],
             )     ,		)

         # Test batched
         SCREAMING_SNAKE_CASE_:				Any							= image_processor(lowerCAmelCase__     ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_videos.shape     ,		(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_frames,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size["height"],
                 self.image_processor_tester.crop_size["width"],
             )     ,		)









import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


lowerCAmelCase							:						Tuple     			=		datasets.utils.logging.get_logger(__name__)



@dataclass
class 						__lowercase       (				datasets.BuilderConfig			):




			"""simple docstring"""







			_UpperCAmelCase     :							Optional[datasets.Features]    							=  None
			_UpperCAmelCase     :							str    							=  "utf-8"
			_UpperCAmelCase     :							Optional[str]    							=  None
			_UpperCAmelCase     :							int    							=  10 << 20  # 10MB
			_UpperCAmelCase     :							bool    							=  False
			_UpperCAmelCase     :							str    							=  "line"







class 						__lowercase       (				datasets.ArrowBasedBuilder			):




			"""simple docstring"""







			_UpperCAmelCase     :							List[Any]    							=  TextConfig



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[Any]):
									return datasets.DatasetInfo(features=self.config.features)



			def  _SCREAMING_SNAKE_CASE    (				self						:  List[Any]     ,		lowerCAmelCase__						:  Tuple):
									if not self.config.data_files:
															raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}")
									SCREAMING_SNAKE_CASE_:				Dict							= dl_manager.download_and_extract(self.config.data_files)
									if isinstance(lowerCAmelCase__     ,		(str, list, tuple)):
															SCREAMING_SNAKE_CASE_:				Dict							= data_files
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				Dict							= [files]
															SCREAMING_SNAKE_CASE_:				List[str]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,		gen_kwargs={"files": files})]
									SCREAMING_SNAKE_CASE_:				Any							= []
									for split_name, files in data_files.items():
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				List[Any]							= [files]
															SCREAMING_SNAKE_CASE_:				Union[str, Any]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															splits.append(datasets.SplitGenerator(name=lowerCAmelCase__     ,		gen_kwargs={"files": files}))
									return splits



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[int]     ,		lowerCAmelCase__						:  pa.Table):
									if self.config.features is not None:
															SCREAMING_SNAKE_CASE_:				Optional[int]							= self.config.features.arrow_schema
															if all(not require_storage_cast(lowerCAmelCase__) for feature in self.config.features.values()):
																					# cheaper cast
																					SCREAMING_SNAKE_CASE_:				List[str]							= pa_table.cast(lowerCAmelCase__)
															else:
																					# more expensive cast; allows str <-> int/float or str to Audio for example
																					SCREAMING_SNAKE_CASE_:				List[Any]							= table_cast(lowerCAmelCase__     ,		lowerCAmelCase__)
															return pa_table
									else:
															return pa_table.cast(pa.schema({"text": pa.string()}))




			def  _SCREAMING_SNAKE_CASE    (				self						:  int     ,		lowerCAmelCase__						:  Dict):
									SCREAMING_SNAKE_CASE_:				str							= list(self.config.features) if self.config.features is not None else ["text"]
									for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase__)):
															# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
															with open(lowerCAmelCase__     ,		encoding=self.config.encoding     ,		errors=self.config.errors) as f:
																					if self.config.sample_by == "line":
																											SCREAMING_SNAKE_CASE_:				Any							= 0
																											while True:
																																	SCREAMING_SNAKE_CASE_:				str							= f.read(self.config.chunksize)
																																	if not batch:
																																							break
																																	batch += f.readline()  # finish current line
																																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= StringIO(lowerCAmelCase__).readlines()
																																	if not self.config.keep_linebreaks:
																																							SCREAMING_SNAKE_CASE_:				Optional[Any]							= [line.rstrip("\n") for line in batch]
																																	SCREAMING_SNAKE_CASE_:				Any							= pa.Table.from_arrays([pa.array(lowerCAmelCase__)]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																					elif self.config.sample_by == "paragraph":
																											SCREAMING_SNAKE_CASE_:				List[str]							= 0
																											SCREAMING_SNAKE_CASE_:				Tuple							= ""
																											while True:
																																	SCREAMING_SNAKE_CASE_:				Dict							= f.read(self.config.chunksize)
																																	if not new_batch:
																																							break
																																	batch += new_batch
																																	batch += f.readline()  # finish current line
																																	SCREAMING_SNAKE_CASE_:				List[str]							= batch.split("\n\n")
																																	SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays(
																																	    [pa.array([example for example in batch[:-1] if example])]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= batch[-1]
																											if batch:
																																	SCREAMING_SNAKE_CASE_:				int							= pa.Table.from_arrays([pa.array([batch])]     ,		names=lowerCAmelCase__)
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																					elif self.config.sample_by == "document":
																											SCREAMING_SNAKE_CASE_:				List[Any]							= f.read()
																											SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays([pa.array([text])]     ,		names=lowerCAmelCase__)
																											yield file_idx, self._cast_table(lowerCAmelCase__)






