import json
import os
from dataclasses import dataclass

import numpy as np
import pyarrow as pa

import datasets
from utils import get_duration


UpperCAmelCase							   =      100000000000
UpperCAmelCase							   =      10000

UpperCAmelCase   ,  UpperCAmelCase							   =      os.path.split(__file__)
UpperCAmelCase							   =      os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))







def 	UpperCAmelCase       (		lowerCAmelCase_						,      lowerCAmelCase_       )	->	datasets.Dataset:
     a  :    Union[str, Any]				=						pa.Table.from_pydict({'''col''': [0] * chunk_size}       )
     a  :    Dict				=						pa.concat_tables([table] * (num_examples // chunk_size)       )
     return datasets.Dataset(lowerCAmelCase_						,      fingerprint='''table_100B'''       )






@dataclass
class 							_SCREAMING_SNAKE_CASE:
   UpperCamelCase_				:						int
   UpperCamelCase_				:						int
   UpperCamelCase_				:						int
   UpperCamelCase_				:						int
   def _lowercase(    self    )	->List[str]:
        a  :    Optional[Any]				=						np.random.default_rng(self.seed    )
        a  :    Dict				=						rng.integers(low=self.low							,     high=self.high							,     size=self.size    ).tolist()
   def __iter__(    self    )	->List[Any]:
        return iter(self._sampled_values    )


   def __len__(    self    )	->List[str]:
        return self.size







@get_duration
def 	UpperCAmelCase       (		lowerCAmelCase_       )	->	str:
     a  :    Dict				=						dataset[0]







@get_duration
def 	UpperCAmelCase       (		lowerCAmelCase_       )	->	List[str]:
     a  :    Dict				=						dataset[-1]







@get_duration
def 	UpperCAmelCase       (		lowerCAmelCase_       )	->	int:
     a  :    str				=						dataset[range(len(lowerCAmelCase_       ) // 2						,      len(lowerCAmelCase_       ) // 2 + 1_0_2_4       )]







@get_duration
def 	UpperCAmelCase       (		lowerCAmelCase_       )	->	Tuple:
     a  :    int				=						dataset[RandIter(0						,      len(lowerCAmelCase_       )						,      1_0_2_4						,      seed=4_2       )]







def 	UpperCAmelCase       (		)	->	Union[str, Any]:
     a  :    Union[str, Any]				=						{'''num examples''': SPEED_TEST_N_EXAMPLES}
     a  :    List[Any]				=						(get_first_row, get_last_row, get_batch_of_aaaa_rows, get_batch_of_aaaa_random_rows)
     print('''generating dataset'''       )
     a  :    List[str]				=						generate_100B_dataset(num_examples=lowerCAmelCase_						,      chunk_size=lowerCAmelCase_       )
     print('''Functions'''       )
     for func in functions:
          print(func.__name__       )
          a  :    str				=						func(lowerCAmelCase_       )

     with open(lowerCAmelCase_						,      '''wb'''       ) as f:
          f.write(json.dumps(lowerCAmelCase_       ).encode('''utf-8'''       )       )


if __name__ == "__main__":  # useful to run the profiler
  benchmark_table_100B()
# this script builds a small sample spm file tests/fixtures/test_sentencepiece_no_bos.model, with features needed by pegasus

# 1. pip install sentencepiece
#
# 2. wget https://raw.githubusercontent.com/google/sentencepiece/master/data/botchan.txt

# 3. build
import sentencepiece as spm

# pegasus:
# 1. no bos
# 2. eos_id is 1
# 3. unk_id is 2
# build a sample spm file accordingly
spm.SentencePieceTrainer.train(
    """--input=botchan.txt --model_prefix=test_sentencepiece_no_bos --bos_id=-1 --unk_id=2  --eos_id=1  --vocab_size=1000"""
)

# 4. now update the fixture
# mv test_sentencepiece_no_bos.model ../../tests/fixtures/