




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 	__lowerCAmelCase			(			__magic_name__   , unittest.TestCase				):
							UpperCamelCase__        =						PreTrainedTokenizerFast
							UpperCamelCase__        =						False
							UpperCamelCase__        =						True
							UpperCamelCase__        =						'''tokenizer_file'''
							def    lowerCamelCase__							(		self		:Dict       ):







								'''simple docstring'''







								a		       =							False  # because we don't have pretrained_vocab_files_map
								super().setUp()
								a		       =							True

								a		       =							["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
								a		       =							"""SaulLu/dummy-tokenizer-bytelevel-bpe"""

								# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
								a		       =							[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

								a		       =							PreTrainedTokenizerFast.from_pretrained(model_paths[0]       )
								tokenizer.save_pretrained(self.tmpdirname       )
							def    lowerCamelCase__							(		self		:int       ):







								'''simple docstring'''







								pass
							def    lowerCamelCase__							(		self		:Any       ):







								'''simple docstring'''







								pass
							def    lowerCamelCase__							(		self		:int       ):







								'''simple docstring'''







								pass
							def    lowerCamelCase__							(		self		:List[str]       ):







								'''simple docstring'''







								pass
							def    lowerCamelCase__							(		self		:Any       ):







								'''simple docstring'''







								a		       =							self.tmpdirname
								# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
										try:
											a		       =							tempfile.mkdtemp()
											a		       =							self.rust_tokenizer_class.from_pretrained(__magic_name__	,   **__magic_name__       )

											tokenizer.save_pretrained(self.tmpdirname       )
											super().test_training_new_tokenizer()
										finally:
											# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
											# is restored
											shutil.rmtree(self.tmpdirname       )
											a		       =							tmpdirname_orig
							def    lowerCamelCase__							(		self		:Union[str, Any]       ):







								'''simple docstring'''







								a		       =							self.tmpdirname
								# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'       ):
										try:
											a		       =							tempfile.mkdtemp()
											a		       =							self.rust_tokenizer_class.from_pretrained(__magic_name__	,   **__magic_name__       )

											tokenizer.save_pretrained(self.tmpdirname       )
											super().test_training_new_tokenizer_with_special_tokens_change()
										finally:
											# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
											# is restored
											shutil.rmtree(self.tmpdirname       )
											a		       =							tmpdirname_orig





							def    lowerCamelCase__							(		self		:str       ):







								'''simple docstring'''







								a		       =							self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name       )

								a		       =							("""a""" for _ in range(1000       ))
								a		       =							tokenizer.train_new_from_iterator(text_iterator=__magic_name__	,   length=1000	,   vocab_size=50       )

								a		       =							new_tokenizer.encode("""a🤗"""       )
								self.assertEqual(__magic_name__	,   [64, 172, 253, 97, 245]       )







@require_tokenizers
class 	__lowerCAmelCase			(			unittest.TestCase				):
							def    lowerCamelCase__							(		self		:Optional[Any]       ):







								'''simple docstring'''







								a		       =							AutoTokenizer.from_pretrained("""bert-base-cased"""       )
								a		       =							json.loads(tokenizer._tokenizer.to_str()       )
								a		       =							len(__magic_name__       )

								with tempfile.TemporaryDirectory() as tmp_dir:
									# Hack to save this in the tokenizer_config.json
									a		       =							["""tokenizer.4.0.0.json"""]
									tokenizer.save_pretrained(__magic_name__       )
									json.dump(__magic_name__	,   open(os.path.join(__magic_name__	,   """tokenizer.4.0.0.json"""       )	,   """w"""       )       )

									# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
									a		       =							AutoTokenizer.from_pretrained(__magic_name__       )
									self.assertEqual(len(__magic_name__       )	,   len(__magic_name__       ) + 1       )
									a		       =							json.loads(new_tokenizer._tokenizer.to_str()       )
									self.assertIn("""huggingface"""	,   json_tokenizer["""model"""]["""vocab"""]       )

									# Will need to be adjusted if we reach v42 and this test is still here.
									# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
									shutil.move(os.path.join(__magic_name__	,   """tokenizer.4.0.0.json"""       )	,   os.path.join(__magic_name__	,   """tokenizer.42.0.0.json"""       )       )
									a		       =							["""tokenizer.42.0.0.json"""]
									tokenizer.save_pretrained(__magic_name__       )
									a		       =							AutoTokenizer.from_pretrained(__magic_name__       )
									self.assertEqual(len(__magic_name__       )	,   len(__magic_name__       )       )
									a		       =							json.loads(new_tokenizer._tokenizer.to_str()       )
									self.assertNotIn("""huggingface"""	,   json_tokenizer["""model"""]["""vocab"""]       )





							def    lowerCamelCase__							(		self		:List[Any]       ):







								'''simple docstring'''







								a		       =							"""hf-internal-testing/test-two-tokenizers"""

								# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
								a		       =							AutoTokenizer.from_pretrained(__magic_name__       )
								self.assertEqual(len(__magic_name__       )	,   2_8997       )
								a		       =							json.loads(tokenizer._tokenizer.to_str()       )
								self.assertIn("""huggingface"""	,   json_tokenizer["""model"""]["""vocab"""]       )

								# Testing an older version by monkey-patching the version in the module it's used.
								import transformers as old_transformers

								a		       =							"""3.0.0"""
								a		       =							old_transformers.models.auto.AutoTokenizer.from_pretrained(__magic_name__       )
								self.assertEqual(len(__magic_name__       )	,   2_8996       )
								a		       =							json.loads(old_tokenizer._tokenizer.to_str()       )
								self.assertNotIn("""huggingface"""	,   json_tokenizer["""model"""]["""vocab"""]       )







@require_tokenizers
class 	__lowerCAmelCase			(			unittest.TestCase				):
							def    lowerCamelCase__							(		self		:Union[str, Any]       ):







								'''simple docstring'''







								a		       =							PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""       )
								a		       =							"""The Matrix is a 1999 science fiction action film."""

								with concurrent.futures.ThreadPoolExecutor() as executor:
									a		       =							[executor.submit(self.fetch	,   __magic_name__	,   __magic_name__       ) for i in range(10       )]
									a		       =							[future.result() for future in futures]
									self.assertEqual(__magic_name__	,   [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10       )]       )





							def    lowerCamelCase__							(		self		:Optional[int]	,   __magic_name__		:str	,   __magic_name__		:Any       ):







								'''simple docstring'''







								return tokenizer.encode(__magic_name__	,   truncation="""longest_first"""	,   padding="""longest"""       )







import os
import tempfile
from pathlib import Path
from unittest import TestCase

import pyarrow as pa
import pytest

from datasets.arrow_dataset import Dataset
from datasets.arrow_reader import ArrowReader, BaseReader, FileInstructions, ReadInstruction, make_file_instructions
from datasets.info import DatasetInfo
from datasets.splits import NamedSplit, Split, SplitDict, SplitInfo

from .utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

class 	__lowerCAmelCase			(			__magic_name__				):
							def    lowerCamelCase__							(		self		:List[str]	,   __magic_name__		:Dict	,   __magic_name__		:Union[str, Any]=False       ):







								'''simple docstring'''







								a       ,				a       ,				a		       =							(
								    filename_skip_take["""filename"""],
								    filename_skip_take["""skip"""] if """skip""" in filename_skip_take else None,
								    filename_skip_take["""take"""] if """take""" in filename_skip_take else None,
								)
								open(os.path.join(__magic_name__       )	,   """wb"""       ).close()
								a		       =							pa.Table.from_pydict({"""filename""": [Path(__magic_name__       ).name] * 100}       )
								if take == -1:
									a		       =							len(__magic_name__       ) - skip
								if skip is not None and take is not None:
									a		       =							pa_table.slice(__magic_name__	,   __magic_name__       )
								return pa_table







class 	__lowerCAmelCase			(			__magic_name__				):
							def    lowerCamelCase__							(		self		:List[Any]       ):







								'''simple docstring'''







								a		       =							"""my_name"""
								a		       =							SplitInfo(name="""train"""	,   num_examples=100       )
								a		       =							SplitInfo(name="""test"""	,   num_examples=100       )
								a		       =							[train_info, test_info]
								a		       =							SplitDict()
								split_dict.add(__magic_name__       )
								split_dict.add(__magic_name__       )
								a		       =							DatasetInfo(splits=__magic_name__       )

								with tempfile.TemporaryDirectory() as tmp_dir:
									a		       =							ReaderTest(__magic_name__	,   __magic_name__       )

									a		       =							"""test[:33%]"""
									a		       =							Dataset(**reader.read(__magic_name__	,   __magic_name__	,   __magic_name__       )       )
									self.assertEqual(dset["""filename"""][0]	,   F'{name}-test'       )
									self.assertEqual(dset.num_rows	,   33       )
									self.assertEqual(dset.num_columns	,   1       )

									a		       =							["""train""", """test[:33%]"""]
									a		       =							[Split.TRAIN, ReadInstruction.from_spec("""test[:33%]"""       )]
									for instructions in [instructionsa, instructionsa]:
										a		       =							[reader.read(__magic_name__	,   __magic_name__	,   __magic_name__       ) for instr in instructions]
										a       ,				a		       =							(Dataset(**__magic_name__       ) for dataset_kwargs in datasets_kwargs)
										self.assertEqual(train_dset["""filename"""][0]	,   F'{name}-train'       )
										self.assertEqual(train_dset.num_rows	,   100       )
										self.assertEqual(train_dset.num_columns	,   1       )
										self.assertIsInstance(train_dset.split	,   __magic_name__       )
										self.assertEqual(str(train_dset.split       )	,   """train"""       )
										self.assertEqual(test_dset["""filename"""][0]	,   F'{name}-test'       )
										self.assertEqual(test_dset.num_rows	,   33       )
										self.assertEqual(test_dset.num_columns	,   1       )
										self.assertIsInstance(test_dset.split	,   __magic_name__       )
										self.assertEqual(str(test_dset.split       )	,   """test[:33%]"""       )
										del train_dset, test_dset
							def    lowerCamelCase__							(		self		:Any       ):







								'''simple docstring'''







								a		       =							"""my_name"""
								a		       =							SplitInfo(name="""train"""	,   num_examples=1000	,   shard_lengths=[100] * 10       )
								a		       =							[train_info]
								a		       =							SplitDict()
								split_dict.add(__magic_name__       )
								a		       =							DatasetInfo(splits=__magic_name__       )

								with tempfile.TemporaryDirectory() as tmp_dir:
									a		       =							ReaderTest(__magic_name__	,   __magic_name__       )

									a		       =							"""train[:33%]"""
									a		       =							Dataset(**reader.read(__magic_name__	,   __magic_name__	,   __magic_name__       )       )
									self.assertEqual(dset["""filename"""][0]	,   F'{name}-train-00000-of-00010'       )
									self.assertEqual(dset["""filename"""][-1]	,   F'{name}-train-00003-of-00010'       )
									self.assertEqual(dset.num_rows	,   330       )
									self.assertEqual(dset.num_columns	,   1       )





							def    lowerCamelCase__							(		self		:List[str]       ):







								'''simple docstring'''







								a		       =							SplitInfo(name="""train"""	,   num_examples=100       )
								a		       =							SplitInfo(name="""test"""	,   num_examples=100       )
								a		       =							SplitDict()
								split_dict.add(__magic_name__       )
								split_dict.add(__magic_name__       )
								a		       =							DatasetInfo(splits=__magic_name__       )

								with tempfile.TemporaryDirectory() as tmp_dir:
									a		       =							ReaderTest(__magic_name__	,   __magic_name__       )

									a		       =							[
									    {"""filename""": os.path.join(__magic_name__	,   """train"""       )},
									    {"""filename""": os.path.join(__magic_name__	,   """test"""       ), """skip""": 10, """take""": 10},
									]
									a		       =							Dataset(**reader.read_files(__magic_name__	,   original_instructions="""train+test[10:20]"""       )       )
									self.assertEqual(dset.num_rows	,   110       )
									self.assertEqual(dset.num_columns	,   1       )
									del dset







@pytest.mark.parametrize("""in_memory"""							,    [False, True]					)
def 	__A	(			__lowerCamelCase							,    __lowerCamelCase							,    __lowerCamelCase					)   -> Optional[int]:
	a		       =							arrow_file
	with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
		a		       =							ArrowReader.read_table(__lowerCamelCase							,    in_memory=__lowerCamelCase					)
	assert table.shape == dataset.data.shape
	assert set(table.column_names					) == set(dataset.data.column_names					)
	assert dict(table.to_pydict()					) == dict(dataset.data.to_pydict()					)  # to_pydict returns OrderedDict





@pytest.mark.parametrize("""in_memory"""							,    [False, True]					)
def 	__A	(			__lowerCamelCase							,    __lowerCamelCase							,    __lowerCamelCase					)   -> str:
	a		       =							arrow_file
	a		       =							ArrowReader(""""""							,    __lowerCamelCase					)
	with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
		a		       =							reader.read_files([{"""filename""": filename}]							,    in_memory=__lowerCamelCase					)
	assert dataset_kwargs.keys() == {"arrow_table", "info", "split"}
	a		       =							dataset_kwargs["""arrow_table"""]
	assert table.shape == dataset.data.shape
	assert set(table.column_names					) == set(dataset.data.column_names					)
	assert dict(table.to_pydict()					) == dict(dataset.data.to_pydict()					)  # to_pydict returns OrderedDict





def 	__A	(			)   -> int:
	assert ReadInstruction("""train"""							,    to=10							,    unit="""abs"""					).to_spec() == "train[:10]"
	assert ReadInstruction("""train"""							,    from_=-80							,    to=10							,    unit="""%"""					).to_spec() == "train[-80%:10%]"

	a		       =							"""train+test"""
	assert ReadInstruction.from_spec(__lowerCamelCase					).to_spec() == spec_train_test

	a		       =							"""train[2:10]"""
	assert ReadInstruction.from_spec(__lowerCamelCase					).to_spec() == spec_train_abs

	a		       =							"""train[15%:-20%]"""
	assert ReadInstruction.from_spec(__lowerCamelCase					).to_spec() == spec_train_pct

	a		       =							"""train[:10%](closest)"""
	assert ReadInstruction.from_spec(__lowerCamelCase					).to_spec() == "train[:10%]"

	a		       =							"""train[:10%](pct1_dropremainder)"""
	assert ReadInstruction.from_spec(__lowerCamelCase					).to_spec() == spec_train_pct_rounding

	a		       =							"""train[:10%](pct1_dropremainder)+test[-10%:](pct1_dropremainder)"""
	assert ReadInstruction.from_spec(__lowerCamelCase					).to_spec() == spec_train_test_pct_rounding





def 	__A	(			)   -> Any:
	a		       =							"""dummy"""
	a		       =							[SplitInfo(name="""train"""							,    num_examples=100					)]
	a		       =							"""train[:33%]"""
	a		       =							"""arrow"""
	a		       =							"""prefix"""

	a		       =							make_file_instructions(__lowerCamelCase							,    __lowerCamelCase							,    __lowerCamelCase							,    __lowerCamelCase							,    __lowerCamelCase					)
	assert isinstance(__lowerCamelCase							,    __lowerCamelCase					)
	assert file_instructions.num_examples == 33
	assert file_instructions.file_instructions == [
	    {"filename": os.path.join(__lowerCamelCase							,    f'{name}-train.arrow'					), "skip": 0, "take": 33}
	]

	a		       =							[SplitInfo(name="""train"""							,    num_examples=100							,    shard_lengths=[10] * 10					)]
	a		       =							make_file_instructions(__lowerCamelCase							,    __lowerCamelCase							,    __lowerCamelCase							,    __lowerCamelCase							,    __lowerCamelCase					)
	assert isinstance(__lowerCamelCase							,    __lowerCamelCase					)
	assert file_instructions.num_examples == 33
	assert file_instructions.file_instructions == [
	    {"filename": os.path.join(__lowerCamelCase							,    f'{name}-train-00000-of-00010.arrow'					), "skip": 0, "take": -1},
	    {"filename": os.path.join(__lowerCamelCase							,    f'{name}-train-00001-of-00010.arrow'					), "skip": 0, "take": -1},
	    {"filename": os.path.join(__lowerCamelCase							,    f'{name}-train-00002-of-00010.arrow'					), "skip": 0, "take": -1},
	    {"filename": os.path.join(__lowerCamelCase							,    f'{name}-train-00003-of-00010.arrow'					), "skip": 0, "take": 3},
	]





@pytest.mark.parametrize("""name, expected_exception"""							,    [(None, TypeError), ("""""", ValueError)]					)
def 	__A	(			__lowerCamelCase							,    __lowerCamelCase					)   -> int:
	a		       =							[SplitInfo(name="""train"""							,    num_examples=100					)]
	a		       =							"""train"""
	a		       =							"""arrow"""
	a		       =							"""prefix_path"""
	with pytest.raises(__lowerCamelCase					):
		a		       =							make_file_instructions(__lowerCamelCase							,    __lowerCamelCase							,    __lowerCamelCase							,    __lowerCamelCase							,    __lowerCamelCase					)

