


import re
import tempfile
import unittest
from pathlib import Path

from datasets.utils.metadata import DatasetMetadata
def 							UpperCamelCase_  (				__snake_case			:						str     )->      str:




						'''simple docstring'''
						__magic_name__ 		=			min(re.search('^ +'	,			__snake_case     ).end() if t.startswith(' '     ) else 0 for t in string.splitlines()     )
						return "\n".join([line[indent_level:] for line in string.splitlines() if indent_level < len(__snake_case     )]     )


UpperCamelCase :				List[Any]						    =       '''\
---
language:
- zh
- en
task_ids:
- sentiment-classification
---
# Begin of markdown

Some cool dataset card
'''

UpperCamelCase :				Optional[Any]						    =       '''\
---
---
# Begin of markdown

Some cool dataset card
'''


UpperCamelCase :				Tuple						    =       '''\
# Begin of markdown

Some cool dataset card
'''




class     __snake_case							(      unittest.TestCase):
					def   a (	self					:	List[Any]     )       ->				List[str]:



											'''simple docstring'''





											with tempfile.TemporaryDirectory() as tmp_dir:
																	__magic_name__ 		=			Path(_lowerCAmelCase     ) / 'README.md'
																	with open(_lowerCAmelCase  ,			'w+'     ) as readme_file:
																							readme_file.write(_lowerCAmelCase     )
																	__magic_name__ 		=			DatasetMetadata.from_readme(_lowerCAmelCase     )
																	self.assertDictEqual(_lowerCAmelCase  ,			{'language': ['zh', 'en'], 'task_ids': ['sentiment-classification']}     )

																	with open(_lowerCAmelCase  ,			'w+'     ) as readme_file:
																							readme_file.write(_lowerCAmelCase     )
																	__magic_name__ 		=			DatasetMetadata.from_readme(_lowerCAmelCase     )
																	self.assertDictEqual(_lowerCAmelCase  ,			{}     )

																	with open(_lowerCAmelCase  ,			'w+'     ) as readme_file:
																							readme_file.write(_lowerCAmelCase     )
																	__magic_name__ 		=			DatasetMetadata.from_readme(_lowerCAmelCase     )
																	self.assertEqual(_lowerCAmelCase  ,			{}     )







					def   a (	self					:	Optional[int]     )       ->				Any:



											'''simple docstring'''





											__magic_name__ 		=			_dedent(
											    '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            '     )
											assert DatasetMetadata.from_yaml_string(_lowerCAmelCase     )

											__magic_name__ 		=			_dedent(
											    '\\n            annotations_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - question-answering\n            task_ids:\n            - open-domain-qa\n            task_ids:\n            - open-domain-qa\n            '     )
											with self.assertRaises(_lowerCAmelCase     ):
																	DatasetMetadata.from_yaml_string(_lowerCAmelCase     )

											__magic_name__ 		=			_dedent(
											    '\\n            annotations_creators:\n            - found\n            language_creators:\n            - found\n            language:\n            - en\n            license:\n            - unknown\n            multilinguality:\n            - monolingual\n            pretty_name: Test Dataset\n            size_categories:\n            - 10K<n<100K\n            source_datasets:\n            - extended|other-yahoo-webscope-l6\n            task_categories:\n            - text-classification\n            task_ids:\n            - multi-class-classification\n            paperswithcode_id:\n            - squad\n            configs:\n            - en\n            train-eval-index:\n            - config: en\n              task: text-classification\n              task_id: multi_class_classification\n              splits:\n                train_split: train\n                eval_split: test\n              col_mapping:\n                text: text\n                label: target\n              metrics:\n                - type: accuracy\n                  name: Accuracy\n            extra_gated_prompt: |\n              By clicking on “Access repository” below, you also agree to ImageNet Terms of Access:\n              [RESEARCHER_FULLNAME] (the "Researcher") has requested permission to use the ImageNet database (the "Database") at Princeton University and Stanford University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:\n              1. Researcher shall use the Database only for non-commercial research and educational purposes.\n            extra_gated_fields:\n              Company: text\n              Country: text\n              I agree to use this model for non-commerical use ONLY: checkbox\n            '     )
											assert DatasetMetadata.from_yaml_string(_lowerCAmelCase     )





import csv
import dataclasses
import json
from dataclasses import dataclass
from typing import List, Optional, Union

from ...utils import is_tf_available, is_torch_available, logging


UpperCamelCase :				Optional[Any]						    =       logging.get_logger(__name__)

@dataclass
class     __snake_case							:
					SCREAMING_SNAKE_CASE_		:					str
					SCREAMING_SNAKE_CASE_		:					str
					SCREAMING_SNAKE_CASE_		:					Optional[str] 				= None
					SCREAMING_SNAKE_CASE_		:					Optional[str] 				= None
					def   a (	self					:	Tuple     )       ->				Dict:



											'''simple docstring'''





											return json.dumps(dataclasses.asdict(self     )  ,			indent=2     ) + "\n"







@dataclass(frozen=_UpperCAmelCase)
class     __snake_case							:
					SCREAMING_SNAKE_CASE_		:					List[int]
					SCREAMING_SNAKE_CASE_		:					Optional[List[int]] 				= None
					SCREAMING_SNAKE_CASE_		:					Optional[List[int]] 				= None
					SCREAMING_SNAKE_CASE_		:					Optional[Union[int, float]] 				= None
					def   a (	self					:	List[str]     )       ->				str:



											'''simple docstring'''





											return json.dumps(dataclasses.asdict(self     )     ) + "\n"







class     __snake_case							:
					def   a (	self					:	Union[str, Any]  ,			_lowerCAmelCase					:	List[str]     )       ->				List[Any]:



											'''simple docstring'''





											raise NotImplementedError()
					def   a (	self					:	Optional[Any]  ,			_lowerCAmelCase					:	str     )       ->				List[Any]:



											'''simple docstring'''





											raise NotImplementedError()
					def   a (	self					:	List[str]  ,			_lowerCAmelCase					:	Optional[Any]     )       ->				Optional[Any]:



											'''simple docstring'''





											raise NotImplementedError()
					def   a (	self					:	Optional[int]  ,			_lowerCAmelCase					:	Union[str, Any]     )       ->				int:



											'''simple docstring'''





											raise NotImplementedError()
					def   a (	self					:	Any     )       ->				int:



											'''simple docstring'''





											raise NotImplementedError()
					def   a (	self					:	int  ,			_lowerCAmelCase					:	Optional[int]     )       ->				Dict:



											'''simple docstring'''





											if len(self.get_labels()     ) > 1:
																	__magic_name__ 		=			self.get_labels()[int(example.label     )]
											return example







					@classmethod
					def   a (	cls					:	Dict  ,			_lowerCAmelCase					:	int  ,			_lowerCAmelCase					:	Optional[Any]=None     )       ->				int:



											'''simple docstring'''





											with open(_lowerCAmelCase  ,			'r'  ,			encoding='utf-8-sig'     ) as f:
																	return list(csv.reader(_lowerCAmelCase  ,			delimiter='\t'  ,			quotechar=_lowerCAmelCase     )     )







class     __snake_case							(      _UpperCAmelCase):
					def __init__(	self					:	List[Any]  ,			_lowerCAmelCase					:	str=None  ,			_lowerCAmelCase					:	Dict=None  ,			_lowerCAmelCase					:	List[Any]="classification"  ,			_lowerCAmelCase					:	List[Any]=False     )       ->				Any:



											'''simple docstring'''





											__magic_name__ 		=			[] if labels is None else labels
											__magic_name__ 		=			[] if examples is None else examples
											__magic_name__ 		=			mode
											__magic_name__ 		=			verbose
					def __len__(	self					:	Any     )       ->				Union[str, Any]:



											'''simple docstring'''





											return len(self.examples     )
					def __getitem__(	self					:	Tuple  ,			_lowerCAmelCase					:	Dict     )       ->				Any:



											'''simple docstring'''





											if isinstance(_lowerCAmelCase  ,			_lowerCAmelCase     ):
																	return SingleSentenceClassificationProcessor(labels=self.labels  ,			examples=self.examples[idx]     )
											return self.examples[idx]
					@classmethod
					def   a (	cls					:	List[Any]  ,			_lowerCAmelCase					:	Tuple  ,			_lowerCAmelCase					:	Union[str, Any]=""  ,			_lowerCAmelCase					:	Optional[Any]=0  ,			_lowerCAmelCase					:	Optional[int]=1  ,			_lowerCAmelCase					:	List[str]=None  ,			_lowerCAmelCase					:	str=False  ,			**_lowerCAmelCase					:	Dict     )       ->				Union[str, Any]:



											'''simple docstring'''





											__magic_name__ 		=			cls(**_lowerCAmelCase     )
											processor.add_examples_from_csv(
											    _lowerCAmelCase  ,			split_name=_lowerCAmelCase  ,			column_label=_lowerCAmelCase  ,			column_text=_lowerCAmelCase  ,			column_id=_lowerCAmelCase  ,			skip_first_row=_lowerCAmelCase  ,			overwrite_labels=_lowerCAmelCase  ,			overwrite_examples=_lowerCAmelCase  ,			)
											return processor
					@classmethod
					def   a (	cls					:	Optional[Any]  ,			_lowerCAmelCase					:	List[Any]  ,			_lowerCAmelCase					:	Optional[Any]=None  ,			**_lowerCAmelCase					:	Union[str, Any]     )       ->				Tuple:



											'''simple docstring'''





											__magic_name__ 		=			cls(**_lowerCAmelCase     )
											processor.add_examples(_lowerCAmelCase  ,			labels=_lowerCAmelCase     )
											return processor
					def   a (	self					:	Union[str, Any]  ,			_lowerCAmelCase					:	str  ,			_lowerCAmelCase					:	Optional[Any]=""  ,			_lowerCAmelCase					:	Optional[Any]=0  ,			_lowerCAmelCase					:	str=1  ,			_lowerCAmelCase					:	Any=None  ,			_lowerCAmelCase					:	List[str]=False  ,			_lowerCAmelCase					:	Dict=False  ,			_lowerCAmelCase					:	Any=False  ,			)       ->				Tuple:



											'''simple docstring'''





											__magic_name__ 		=			self._read_tsv(_lowerCAmelCase     )
											if skip_first_row:
																	__magic_name__ 		=			lines[1:]
											__magic_name__ 		=			[]
											__magic_name__ 		=			[]
											__magic_name__ 		=			[]
											for i, line in enumerate(_lowerCAmelCase     ):
																	texts.append(line[column_text]     )
																	labels.append(line[column_label]     )
																	if column_id is not None:
																							ids.append(line[column_id]     )
																	else:
																							__magic_name__ 		=			F'''{split_name}-{i}''' if split_name else str(_lowerCAmelCase     )
																							ids.append(_lowerCAmelCase     )

											return self.add_examples(
											    _lowerCAmelCase  ,			_lowerCAmelCase  ,			_lowerCAmelCase  ,			overwrite_labels=_lowerCAmelCase  ,			overwrite_examples=_lowerCAmelCase     )
					def   a (	self					:	Any  ,			_lowerCAmelCase					:	Optional[int]  ,			_lowerCAmelCase					:	Optional[Any]=None  ,			_lowerCAmelCase					:	Any=None  ,			_lowerCAmelCase					:	Optional[int]=False  ,			_lowerCAmelCase					:	List[str]=False     )       ->				Dict:



											'''simple docstring'''





											if labels is not None and len(_lowerCAmelCase     ) != len(_lowerCAmelCase     ):
																	raise ValueError(
																	    F'''Text and labels have mismatched lengths {len(_lowerCAmelCase     )} and {len(_lowerCAmelCase     )}'''     )
											if ids is not None and len(_lowerCAmelCase     ) != len(_lowerCAmelCase     ):
																	raise ValueError(F'''Text and ids have mismatched lengths {len(_lowerCAmelCase     )} and {len(_lowerCAmelCase     )}'''     )
											if ids is None:
																	__magic_name__ 		=			[None] * len(_lowerCAmelCase     )
											if labels is None:
																	__magic_name__ 		=			[None] * len(_lowerCAmelCase     )
											__magic_name__ 		=			[]
											__magic_name__ 		=			set()
											for text_or_text_and_label, label, guid in zip(_lowerCAmelCase  ,			_lowerCAmelCase  ,			_lowerCAmelCase     ):
																	if isinstance(_lowerCAmelCase  ,			(tuple, list)     ) and label is None:
																							__magic_name__		,			__magic_name__ 		=			text_or_text_and_label
																	else:
																							__magic_name__ 		=			text_or_text_and_label
																	added_labels.add(_lowerCAmelCase     )
																	examples.append(InputExample(guid=_lowerCAmelCase  ,			text_a=_lowerCAmelCase  ,			text_b=_lowerCAmelCase  ,			label=_lowerCAmelCase     )     )

											# Update examples
											if overwrite_examples:
																	__magic_name__ 		=			examples
											else:
																	self.examples.extend(_lowerCAmelCase     )

											# Update labels
											if overwrite_labels:
																	__magic_name__ 		=			list(_lowerCAmelCase     )
											else:
																	__magic_name__ 		=			list(set(self.labels     ).union(_lowerCAmelCase     )     )

											return self.examples







					def   a (	self					:	Optional[int]  ,			_lowerCAmelCase					:	Tuple  ,			_lowerCAmelCase					:	Optional[int]=None  ,			_lowerCAmelCase					:	Dict=False  ,			_lowerCAmelCase					:	Tuple=0  ,			_lowerCAmelCase					:	List[str]=True  ,			_lowerCAmelCase					:	List[str]=None  ,			)       ->				Dict:



											'''simple docstring'''





											if max_length is None:
																	__magic_name__ 		=			tokenizer.max_len

											__magic_name__ 		=			{label: i for i, label in enumerate(self.labels     )}

											__magic_name__ 		=			[]
											for ex_index, example in enumerate(self.examples     ):
																	if ex_index % 1_0_0_0_0 == 0:
																							logger.info(F'''Tokenizing example {ex_index}'''     )

																	__magic_name__ 		=			tokenizer.encode(
																	    example.text_a  ,			add_special_tokens=_lowerCAmelCase  ,			max_length=min(_lowerCAmelCase  ,			tokenizer.max_len     )  ,			)
																	all_input_ids.append(_lowerCAmelCase     )

											__magic_name__ 		=			max(len(_lowerCAmelCase     ) for input_ids in all_input_ids     )

											__magic_name__ 		=			[]
											for ex_index, (input_ids, example) in enumerate(zip(_lowerCAmelCase  ,			self.examples     )     ):
																	if ex_index % 1_0_0_0_0 == 0:
																							logger.info(F'''Writing example {ex_index}/{len(self.examples     )}'''     )
																	# The mask has 1 for real tokens and 0 for padding tokens. Only real
																	# tokens are attended to.
																	__magic_name__ 		=			[1 if mask_padding_with_zero else 0] * len(_lowerCAmelCase     )

																	# Zero-pad up to the sequence length.
																	__magic_name__ 		=			batch_length - len(_lowerCAmelCase     )
																	if pad_on_left:
																							__magic_name__ 		=			([pad_token] * padding_length) + input_ids
																							__magic_name__ 		=			([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
																	else:
																							__magic_name__ 		=			input_ids + ([pad_token] * padding_length)
																							__magic_name__ 		=			attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)

																	if len(_lowerCAmelCase     ) != batch_length:
																							raise ValueError(F'''Error with input length {len(_lowerCAmelCase     )} vs {batch_length}'''     )
																	if len(_lowerCAmelCase     ) != batch_length:
																							raise ValueError(F'''Error with input length {len(_lowerCAmelCase     )} vs {batch_length}'''     )

																	if self.mode == "classification":
																							__magic_name__ 		=			label_map[example.label]
																	elif self.mode == "regression":
																							__magic_name__ 		=			float(example.label     )
																	else:
																							raise ValueError(self.mode     )

																	if ex_index < 5 and self.verbose:
																							logger.info('*** Example ***'     )
																							logger.info(F'''guid: {example.guid}'''     )
																							logger.info(F'''input_ids: {" ".join([str(_lowerCAmelCase     ) for x in input_ids]     )}'''     )
																							logger.info(F'''attention_mask: {" ".join([str(_lowerCAmelCase     ) for x in attention_mask]     )}'''     )
																							logger.info(F'''label: {example.label} (id = {label})'''     )

																	features.append(InputFeatures(input_ids=_lowerCAmelCase  ,			attention_mask=_lowerCAmelCase  ,			label=_lowerCAmelCase     )     )

											if return_tensors is None:
																	return features
											elif return_tensors == "tf":
																	if not is_tf_available():
																							raise RuntimeError('return_tensors set to \'tf\' but TensorFlow 2.0 can\'t be imported'     )
																	import tensorflow as tf

																	def gen():
																							for ex in features:
																													yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)

																	__magic_name__ 		=			tf.data.Dataset.from_generator(
																	    _lowerCAmelCase  ,			({'input_ids': tf.intaa, 'attention_mask': tf.intaa}, tf.intaa)  ,			({'input_ids': tf.TensorShape([None]     ), 'attention_mask': tf.TensorShape([None]     )}, tf.TensorShape([]     ))  ,			)
																	return dataset
											elif return_tensors == "pt":
																	if not is_torch_available():
																							raise RuntimeError('return_tensors set to \'pt\' but PyTorch can\'t be imported'     )
																	import torch
																	from torch.utils.data import TensorDataset

																	__magic_name__ 		=			torch.tensor([f.input_ids for f in features]  ,			dtype=torch.long     )
																	__magic_name__ 		=			torch.tensor([f.attention_mask for f in features]  ,			dtype=torch.long     )
																	if self.mode == "classification":
																							__magic_name__ 		=			torch.tensor([f.label for f in features]  ,			dtype=torch.long     )
																	elif self.mode == "regression":
																							__magic_name__ 		=			torch.tensor([f.label for f in features]  ,			dtype=torch.float     )

																	__magic_name__ 		=			TensorDataset(_lowerCAmelCase  ,			_lowerCAmelCase  ,			_lowerCAmelCase     )
																	return dataset
											else:
																	raise ValueError('return_tensors should be one of \'tf\' or \'pt\''     )

