





import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode





def 							__lowerCAmelCase(					_SCREAMING_SNAKE_CASE       ,    **_SCREAMING_SNAKE_CASE							)       ->   Any:




    """simple docstring"""




    return RunBeamCommand(
        args.dataset       ,    args.name       ,    args.cache_dir       ,    args.beam_pipeline_options       ,    args.data_dir       ,    args.all_configs       ,    args.save_info or args.save_infos       ,    args.ignore_verifications       ,    args.force_redownload       ,    **_SCREAMING_SNAKE_CASE       ,    )







class     lowerCamelCase(	__snake_case  ):
     '''simple docstring'''


     @staticmethod
     def 					lowerCAmelCase__		(						snake_case_       ):
         _A    =       parser.add_parser('run_beam'							,    help='Run a Beam dataset processing pipeline'       )
         run_beam_parser.add_argument('dataset'							,    type=snake_case_							,    help='Name of the dataset to download'       )
         run_beam_parser.add_argument('--name'							,    type=snake_case_							,    default=snake_case_							,    help='Dataset config name'       )
         run_beam_parser.add_argument(
             '--cache_dir'							,    type=snake_case_							,    default=snake_case_							,    help='Cache directory where the datasets are stored'							,    )
         run_beam_parser.add_argument(
             '--beam_pipeline_options'							,    type=snake_case_							,    default=''							,    help='Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`'							,    )
         run_beam_parser.add_argument(
             '--data_dir'							,    type=snake_case_							,    default=snake_case_							,    help='Can be used to specify a manual directory to get the files from'							,    )
         run_beam_parser.add_argument('--all_configs'							,    action='store_true'							,    help='Test all dataset configurations'       )
         run_beam_parser.add_argument('--save_info'							,    action='store_true'							,    help='Save the dataset infos file'       )
         run_beam_parser.add_argument(
             '--ignore_verifications'							,    action='store_true'							,    help='Run the test without checksums and splits checks'       )
         run_beam_parser.add_argument('--force_redownload'							,    action='store_true'							,    help='Force dataset redownload'       )
         # aliases
         run_beam_parser.add_argument('--save_infos'							,    action='store_true'							,    help='alias for save_info'       )
         run_beam_parser.set_defaults(func=snake_case_       )




     def __init__(						self							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    snake_case_							,    **snake_case_							,    ):
         _A    =       dataset
         _A    =       name
         _A    =       cache_dir
         _A    =       beam_pipeline_options
         _A    =       data_dir
         _A    =       all_configs
         _A    =       save_infos
         _A    =       ignore_verifications
         _A    =       force_redownload
         _A    =       config_kwargs




     def 					lowerCAmelCase__		(						self       ):
         import apache_beam as beam

         if self._name is not None and self._all_configs:
             print('Both parameters `name` and `all_configs` can\'t be used at once.'       )
             exit(1       )
         _A, _A    =       self._dataset, self._name
         _A    =       dataset_module_factory(snake_case_       )
         _A    =       import_main_class(dataset_module.module_path       )
         _A    =       []
         if self._beam_pipeline_options:
             _A    =       beam.options.pipeline_options.PipelineOptions(
                 flags=[F"--{opt.strip()}" for opt in self._beam_pipeline_options.split(','       ) if opt]       )
         else:
             _A    =       None
         if self._all_configs and len(builder_cls.BUILDER_CONFIGS       ) > 0:
             for builder_config in builder_cls.BUILDER_CONFIGS:
                 builders.append(
                     builder_cls(
                         config_name=builder_config.name							,    data_dir=self._data_dir							,    hash=dataset_module.hash							,    beam_options=snake_case_							,    cache_dir=self._cache_dir							,    base_path=dataset_module.builder_kwargs.get('base_path'       )							,    )       )
         else:
             builders.append(
                 builder_cls(
                     config_name=snake_case_							,    data_dir=self._data_dir							,    beam_options=snake_case_							,    cache_dir=self._cache_dir							,    base_path=dataset_module.builder_kwargs.get('base_path'       )							,    **self._config_kwargs							,    )       )

         for builder in builders:
             builder.download_and_prepare(
                 download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
                 if not self._force_redownload
                 else DownloadMode.FORCE_REDOWNLOAD							,    download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH       )							,    verification_mode=VerificationMode.NO_CHECKS
                 if self._ignore_verifications
                 else VerificationMode.ALL_CHECKS							,    try_from_hf_gcs=snake_case_							,    )
             if self._save_infos:
                 builder._save_infos()

         print('Apache beam run successful.'       )

         # If save_infos=True, the dataset infos file is created next to the loaded module file.
         # Let's move it to the original directory of the dataset script, to allow the user to
         # upload them on S3 at the same time afterwards.
         if self._save_infos:
             _A    =       os.path.join(builder_cls.get_imported_module_dir()							,    config.DATASETDICT_INFOS_FILENAME       )

             _A    =       Path(snake_case_       ).name + '.py'

             _A    =       os.path.join(snake_case_							,    snake_case_       )
             if os.path.isfile(snake_case_       ):
                 _A    =       os.path.dirname(snake_case_       )
             elif os.path.isfile(snake_case_       ):
                 _A    =       path
             else:  # in case of a remote dataset
                 print(F"Dataset Infos file saved at {dataset_infos_path}"       )
                 exit(1       )

             # Move datasetinfo back to the user
             _A    =       os.path.join(snake_case_							,    config.DATASETDICT_INFOS_FILENAME       )
             copyfile(snake_case_							,    snake_case_       )
             print(F"Dataset Infos file saved at {user_dataset_infos_path}"       )












import csv
import dataclasses
import json
from dataclasses import dataclass
from typing import List, Optional, Union

from ...utils import is_tf_available, is_torch_available, logging


__A       : Optional[int]						    = logging.get_logger(__name__)







@dataclass
class     lowerCamelCase:
     '''simple docstring'''


     __magic_name__        = 42
     __magic_name__        = 42
     __magic_name__        = None
     __magic_name__        = None




     def 					lowerCAmelCase__		(						self       ):
         return json.dumps(dataclasses.asdict(self       )							,    indent=2       ) + "\n"







@dataclass(frozen=__snake_case  )
class     lowerCamelCase:
     '''simple docstring'''


     __magic_name__        = 42
     __magic_name__        = None
     __magic_name__        = None
     __magic_name__        = None




     def 					lowerCAmelCase__		(						self       ):
         return json.dumps(dataclasses.asdict(self       )       ) + "\n"







class     lowerCamelCase:
     '''simple docstring'''


     def 					lowerCAmelCase__		(						self							,    snake_case_       ):
         raise NotImplementedError()




     def 					lowerCAmelCase__		(						self							,    snake_case_       ):
         raise NotImplementedError()




     def 					lowerCAmelCase__		(						self							,    snake_case_       ):
         raise NotImplementedError()




     def 					lowerCAmelCase__		(						self							,    snake_case_       ):
         raise NotImplementedError()




     def 					lowerCAmelCase__		(						self       ):
         raise NotImplementedError()




     def 					lowerCAmelCase__		(						self							,    snake_case_       ):
         if len(self.get_labels()       ) > 1:
             _A    =       self.get_labels()[int(example.label       )]
         return example




     @classmethod
     def 					lowerCAmelCase__		(						cls							,    snake_case_							,    snake_case_=None       ):
         with open(snake_case_							,    'r'							,    encoding='utf-8-sig'       ) as f:
             return list(csv.reader(snake_case_							,    delimiter='\t'							,    quotechar=snake_case_       )       )







class     lowerCamelCase(	__snake_case  ):
     '''simple docstring'''


     def __init__(						self							,    snake_case_=None							,    snake_case_=None							,    snake_case_="classification"							,    snake_case_=False       ):
         _A    =       [] if labels is None else labels
         _A    =       [] if examples is None else examples
         _A    =       mode
         _A    =       verbose




     def __len__(						self       ):
         return len(self.examples       )




     def __getitem__(						self							,    snake_case_       ):
         if isinstance(snake_case_							,    snake_case_       ):
             return SingleSentenceClassificationProcessor(labels=self.labels							,    examples=self.examples[idx]       )
         return self.examples[idx]




     @classmethod
     def 					lowerCAmelCase__		(						cls							,    snake_case_							,    snake_case_=""							,    snake_case_=0							,    snake_case_=1							,    snake_case_=None							,    snake_case_=False							,    **snake_case_       ):
         _A    =       cls(**snake_case_       )
         processor.add_examples_from_csv(
             snake_case_							,    split_name=snake_case_							,    column_label=snake_case_							,    column_text=snake_case_							,    column_id=snake_case_							,    skip_first_row=snake_case_							,    overwrite_labels=snake_case_							,    overwrite_examples=snake_case_							,    )
         return processor




     @classmethod
     def 					lowerCAmelCase__		(						cls							,    snake_case_							,    snake_case_=None							,    **snake_case_       ):
         _A    =       cls(**snake_case_       )
         processor.add_examples(snake_case_							,    labels=snake_case_       )
         return processor




     def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_=""							,    snake_case_=0							,    snake_case_=1							,    snake_case_=None							,    snake_case_=False							,    snake_case_=False							,    snake_case_=False							,    ):
         _A    =       self._read_tsv(snake_case_       )
         if skip_first_row:
             _A    =       lines[1:]
         _A    =       []
         _A    =       []
         _A    =       []
         for i, line in enumerate(snake_case_       ):
             texts.append(line[column_text]       )
             labels.append(line[column_label]       )
             if column_id is not None:
                 ids.append(line[column_id]       )
             else:
                 _A    =       F"{split_name}-{i}" if split_name else str(snake_case_       )
                 ids.append(snake_case_       )

         return self.add_examples(
             snake_case_							,    snake_case_							,    snake_case_							,    overwrite_labels=snake_case_							,    overwrite_examples=snake_case_       )




     def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_=None							,    snake_case_=None							,    snake_case_=False							,    snake_case_=False       ):
         if labels is not None and len(snake_case_       ) != len(snake_case_       ):
             raise ValueError(
                 F"Text and labels have mismatched lengths {len(snake_case_       )} and {len(snake_case_       )}"       )
         if ids is not None and len(snake_case_       ) != len(snake_case_       ):
             raise ValueError(F"Text and ids have mismatched lengths {len(snake_case_       )} and {len(snake_case_       )}"       )
         if ids is None:
             _A    =       [None] * len(snake_case_       )
         if labels is None:
             _A    =       [None] * len(snake_case_       )
         _A    =       []
         _A    =       set()
         for text_or_text_and_label, label, guid in zip(snake_case_							,    snake_case_							,    snake_case_       ):
             if isinstance(snake_case_							,    (tuple, list)       ) and label is None:
                 _A, _A    =       text_or_text_and_label
             else:
                 _A    =       text_or_text_and_label
             added_labels.add(snake_case_       )
             examples.append(InputExample(guid=snake_case_							,    text_a=snake_case_							,    text_b=snake_case_							,    label=snake_case_       )       )

         # Update examples
         if overwrite_examples:
             _A    =       examples
         else:
             self.examples.extend(snake_case_       )

         # Update labels
         if overwrite_labels:
             _A    =       list(snake_case_       )
         else:
             _A    =       list(set(self.labels       ).union(snake_case_       )       )

         return self.examples




     def 					lowerCAmelCase__		(						self							,    snake_case_							,    snake_case_=None							,    snake_case_=False							,    snake_case_=0							,    snake_case_=True							,    snake_case_=None							,    ):
         if max_length is None:
             _A    =       tokenizer.max_len

         _A    =       {label: i for i, label in enumerate(self.labels       )}

         _A    =       []
         for ex_index, example in enumerate(self.examples       ):
             if ex_index % 1_0000 == 0:
                 logger.info(F"Tokenizing example {ex_index}"       )

             _A    =       tokenizer.encode(
                 example.text_a							,    add_special_tokens=snake_case_							,    max_length=min(snake_case_							,    tokenizer.max_len       )							,    )
             all_input_ids.append(snake_case_       )

         _A    =       max(len(snake_case_       ) for input_ids in all_input_ids       )

         _A    =       []
         for ex_index, (input_ids, example) in enumerate(zip(snake_case_							,    self.examples       )       ):
             if ex_index % 1_0000 == 0:
                 logger.info(F"Writing example {ex_index}/{len(self.examples       )}"       )
             # The mask has 1 for real tokens and 0 for padding tokens. Only real
             # tokens are attended to.
             _A    =       [1 if mask_padding_with_zero else 0] * len(snake_case_       )

             # Zero-pad up to the sequence length.
             _A    =       batch_length - len(snake_case_       )
             if pad_on_left:
                 _A    =       ([pad_token] * padding_length) + input_ids
                 _A    =       ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
             else:
                 _A    =       input_ids + ([pad_token] * padding_length)
                 _A    =       attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)

             if len(snake_case_       ) != batch_length:
                 raise ValueError(F"Error with input length {len(snake_case_       )} vs {batch_length}"       )
             if len(snake_case_       ) != batch_length:
                 raise ValueError(F"Error with input length {len(snake_case_       )} vs {batch_length}"       )

             if self.mode == "classification":
                 _A    =       label_map[example.label]
             elif self.mode == "regression":
                 _A    =       float(example.label       )
             else:
                 raise ValueError(self.mode       )

             if ex_index < 5 and self.verbose:
                 logger.info('*** Example ***'       )
                 logger.info(F"guid: {example.guid}"       )
                 logger.info(F"input_ids: {' '.join([str(snake_case_       ) for x in input_ids]       )}"       )
                 logger.info(F"attention_mask: {' '.join([str(snake_case_       ) for x in attention_mask]       )}"       )
                 logger.info(F"label: {example.label} (id = {label})"       )

             features.append(InputFeatures(input_ids=snake_case_							,    attention_mask=snake_case_							,    label=snake_case_       )       )

         if return_tensors is None:
             return features
         elif return_tensors == "tf":
             if not is_tf_available():
                 raise RuntimeError('return_tensors set to \'tf\' but TensorFlow 2.0 can\'t be imported'       )
             import tensorflow as tf

             def gen():
                 for ex in features:
                     yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)

             _A    =       tf.data.Dataset.from_generator(
                 snake_case_							,    ({'input_ids': tf.intaa, 'attention_mask': tf.intaa}, tf.intaa)							,    ({'input_ids': tf.TensorShape([None]       ), 'attention_mask': tf.TensorShape([None]       )}, tf.TensorShape([]       ))							,    )
             return dataset
         elif return_tensors == "pt":
             if not is_torch_available():
                 raise RuntimeError('return_tensors set to \'pt\' but PyTorch can\'t be imported'       )
             import torch
             from torch.utils.data import TensorDataset

             _A    =       torch.tensor([f.input_ids for f in features]							,    dtype=torch.long       )
             _A    =       torch.tensor([f.attention_mask for f in features]							,    dtype=torch.long       )
             if self.mode == "classification":
                 _A    =       torch.tensor([f.label for f in features]							,    dtype=torch.long       )
             elif self.mode == "regression":
                 _A    =       torch.tensor([f.label for f in features]							,    dtype=torch.float       )

             _A    =       TensorDataset(snake_case_							,    snake_case_							,    snake_case_       )
             return dataset
         else:
             raise ValueError('return_tensors should be one of \'tf\' or \'pt\''       )





