


"""simple docstring"""





import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class  lowercase_  ( a_						):







      def __init__(      self :							List[str]		,  _lowercase :							str		,  _lowercase :							List[Any] = None		,  _lowercase :							Optional[int] = None		,  _lowercase :							List[str] = None		,  _lowercase :							Tuple = False		,  _lowercase :							str = False		,  _lowercase :							Union[str, Any] = None		,  **_lowercase :							Dict		,  ):
        super().__init__(
            _lowercase		,  split=_lowercase		,  features=_lowercase		,  cache_dir=_lowercase		,  keep_in_memory=_lowercase		,  streaming=_lowercase		,  num_proc=_lowercase		,  **_lowercase		,  )
        lowerCAmelCase__    :    Any								=					path_or_paths if isinstance(_lowercase		,  _lowercase    ) else {self.split: path_or_paths}
        lowerCAmelCase__    :    Optional[Any]								=					Csv(
            cache_dir=_lowercase		,  data_files=_lowercase		,  features=_lowercase		,  **_lowercase		,  )







      def 				_lowerCAmelCase       (      self :							List[str]    ):






        if self.streaming:
          lowerCAmelCase__    :    str								=					self.builder.as_streaming_dataset(split=self.split    )
        # Build regular (map-style) dataset
        else:
          lowerCAmelCase__    :    int								=					None
          lowerCAmelCase__    :    Dict								=					None
          lowerCAmelCase__    :    Dict								=					None
          lowerCAmelCase__    :    Union[str, Any]								=					None

          self.builder.download_and_prepare(
              download_config=_lowercase		,  download_mode=_lowercase		,  verification_mode=_lowercase		,  base_path=_lowercase		,  num_proc=self.num_proc		,  )
          lowerCAmelCase__    :    Dict								=					self.builder.as_dataset(
              split=self.split		,  verification_mode=_lowercase		,  in_memory=self.keep_in_memory    )
        return dataset

class  lowercase_  :







      def __init__(      self :							str		,  _lowercase :							Union[str, Any]		,  _lowercase :							int		,  _lowercase :							str = None		,  _lowercase :							str = None		,  **_lowercase :							Union[str, Any]		,  ):






        if num_proc is not None and num_proc <= 0:
          raise ValueError(f"num_proc {num_proc} must be an integer > 0."    )

        lowerCAmelCase__    :    Dict								=					dataset
        lowerCAmelCase__    :    Optional[Any]								=					path_or_buf
        lowerCAmelCase__    :    List[str]								=					batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
        lowerCAmelCase__    :    Optional[Any]								=					num_proc
        lowerCAmelCase__    :    int								=					"utf-8"
        lowerCAmelCase__    :    List[Any]								=					to_csv_kwargs







      def 				_lowerCAmelCase       (      self :							str    ):
        lowerCAmelCase__    :    int								=					self.to_csv_kwargs.pop("path_or_buf"		,  _lowercase    )
        lowerCAmelCase__    :    Optional[Any]								=					self.to_csv_kwargs.pop("header"		,  _lowercase    )
        lowerCAmelCase__    :    Union[str, Any]								=					self.to_csv_kwargs.pop("index"		,  _lowercase    )

        if isinstance(self.path_or_buf		,  (str, bytes, os.PathLike)    ):
          with open(self.path_or_buf		,  "wb+"    ) as buffer:
            lowerCAmelCase__    :    str								=					self._write(file_obj=_lowercase		,  header=_lowercase		,  index=_lowercase		,  **self.to_csv_kwargs    )
        else:
          lowerCAmelCase__    :    Union[str, Any]								=					self._write(file_obj=self.path_or_buf		,  header=_lowercase		,  index=_lowercase		,  **self.to_csv_kwargs    )
        return written







      def 				_lowerCAmelCase       (      self :							Tuple		,  _lowercase :							Optional[int]    ):
        lowerCAmelCase__    :    Optional[int]								=					args

        lowerCAmelCase__    :    Optional[Any]								=					query_table(
            table=self.dataset.data		,  key=slice(_lowercase		,  offset + self.batch_size    )		,  indices=self.dataset._indices		,  )
        lowerCAmelCase__    :    Dict								=					batch.to_pandas().to_csv(
            path_or_buf=_lowercase		,  header=header if (offset == 0) else False		,  index=_lowercase		,  **_lowercase    )
        return csv_str.encode(self.encoding    )







      def 				_lowerCAmelCase       (      self :							Optional[Any]		,  _lowercase :							Union[str, Any]		,  _lowercase :							str		,  _lowercase :							Optional[int]		,  **_lowercase :							Union[str, Any]    ):
        lowerCAmelCase__    :    List[Any]								=					0

        if self.num_proc is None or self.num_proc == 1:
          for offset in logging.tqdm(
              range(0		,  len(self.dataset    )		,  self.batch_size    )		,  unit="ba"		,  disable=not logging.is_progress_bar_enabled()		,  desc="Creating CSV from Arrow format"		,  ):
            lowerCAmelCase__    :    int								=					self._batch_csv((offset, header, index, to_csv_kwargs)    )
            written += file_obj.write(_lowercase    )

        else:
          lowerCAmelCase__    :    Dict								=					len(self.dataset    ), self.batch_size
          with multiprocessing.Pool(self.num_proc    ) as pool:
            for csv_str in logging.tqdm(
                pool.imap(
                    self._batch_csv		,  [(offset, header, index, to_csv_kwargs) for offset in range(0		,  _lowercase		,  _lowercase    )]		,  )		,  total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,  unit="ba"		,  disable=not logging.is_progress_bar_enabled()		,  desc="Creating CSV from Arrow format"		,  ):
              written += file_obj.write(_lowercase    )

        return written




"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase    				=      logging.get_logger(__name__)

__UpperCAmelCase    				=      {
    "MCG-NJU/videomae-base": "https://huggingface.co/MCG-NJU/videomae-base/resolve/main/config.json",
}

class  lowercase_  ( a_						):
      __magic_name__     :							Union[str, Any]          =				"""videomae"""







      def __init__(      self :							Dict		,  _lowercase :							int=2_2_4		,  _lowercase :							Optional[Any]=1_6		,  _lowercase :							List[Any]=3		,  _lowercase :							int=1_6		,  _lowercase :							Optional[Any]=2		,  _lowercase :							Union[str, Any]=7_6_8		,  _lowercase :							int=1_2		,  _lowercase :							Dict=1_2		,  _lowercase :							Optional[Any]=3_0_7_2		,  _lowercase :							List[str]="gelu"		,  _lowercase :							int=0.0		,  _lowercase :							int=0.0		,  _lowercase :							Optional[int]=0.02		,  _lowercase :							List[str]=1e-1_2		,  _lowercase :							Union[str, Any]=True		,  _lowercase :							Union[str, Any]=True		,  _lowercase :							Any=6		,  _lowercase :							Any=3_8_4		,  _lowercase :							int=4		,  _lowercase :							Optional[int]=1_5_3_6		,  _lowercase :							Tuple=True		,  **_lowercase :							List[str]		,  ):
        super().__init__(**_lowercase    )

        lowerCAmelCase__    :    List[Any]								=					image_size
        lowerCAmelCase__    :    Tuple								=					patch_size
        lowerCAmelCase__    :    Union[str, Any]								=					num_channels
        lowerCAmelCase__    :    Optional[Any]								=					num_frames
        lowerCAmelCase__    :    Union[str, Any]								=					tubelet_size

        lowerCAmelCase__    :    Union[str, Any]								=					hidden_size
        lowerCAmelCase__    :    Dict								=					num_hidden_layers
        lowerCAmelCase__    :    Any								=					num_attention_heads
        lowerCAmelCase__    :    List[Any]								=					intermediate_size
        lowerCAmelCase__    :    Optional[Any]								=					hidden_act
        lowerCAmelCase__    :    int								=					hidden_dropout_prob
        lowerCAmelCase__    :    Optional[Any]								=					attention_probs_dropout_prob
        lowerCAmelCase__    :    Any								=					initializer_range
        lowerCAmelCase__    :    Any								=					layer_norm_eps
        lowerCAmelCase__    :    Union[str, Any]								=					qkv_bias
        lowerCAmelCase__    :    int								=					use_mean_pooling

        lowerCAmelCase__    :    List[Any]								=					decoder_num_attention_heads
        lowerCAmelCase__    :    List[str]								=					decoder_hidden_size
        lowerCAmelCase__    :    int								=					decoder_num_hidden_layers
        lowerCAmelCase__    :    Optional[int]								=					decoder_intermediate_size
        lowerCAmelCase__    :    Optional[Any]								=					norm_pix_loss
