




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__magic_name__     :     Union[str, Any]  							=					logging.get_logger(__name__)

__magic_name__     :     Any  							=					{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

__magic_name__     :     Union[str, Any]  							=					{
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

__magic_name__     :     Optional[int]  							=					{
    """openai-gpt""": 5_12,
}
def 		a_       (   __lowerCAmelCase							):
  lowerCAmelCase__								=			text.strip()
  if not text:
    return []
  lowerCAmelCase__								=			text.split()
  return tokens





class 						SCREAMING_SNAKE_CASE__		(_a ):
     def __init__(	self  :		Any					,	__lowerCamelCase  :		str=True					,	__lowerCamelCase  :		List[str]=None					,	__lowerCamelCase  :		List[Any]=True					,	__lowerCamelCase  :		Optional[Any]=None					,	__lowerCamelCase  :		List[str]=True					,	):



       """simple docstring"""
       if never_split is None:
         lowerCAmelCase__								=			[]
       lowerCAmelCase__								=			do_lower_case
       lowerCAmelCase__								=			set(__lowerCamelCase	)
       lowerCAmelCase__								=			tokenize_chinese_chars
       lowerCAmelCase__								=			strip_accents
       lowerCAmelCase__								=			do_split_on_punc
     def 					A__			(	self  :		List[Any]					,	__lowerCamelCase  :		List[Any]					,	__lowerCamelCase  :		Dict=None	):



       """simple docstring"""
       lowerCAmelCase__								=			self.never_split.union(set(__lowerCamelCase	)	) if never_split else self.never_split
       lowerCAmelCase__								=			self._clean_text(__lowerCamelCase	)

       # This was added on November 1st, 2018 for the multilingual and Chinese
       # models. This is also applied to the English models now, but it doesn't
       # matter since the English models were not trained on any Chinese data
       # and generally don't have any Chinese data in them (there are Chinese
       # characters in the vocabulary because Wikipedia does have some Chinese
       # words in the English Wikipedia.).
       if self.tokenize_chinese_chars:
         lowerCAmelCase__								=			self._tokenize_chinese_chars(__lowerCamelCase	)
       # prevents treating the same character with different unicode codepoints as different characters
       lowerCAmelCase__								=			unicodedata.normalize('''NFC'''					,	__lowerCamelCase	)
       lowerCAmelCase__								=			whitespace_tokenize(__lowerCamelCase	)
       lowerCAmelCase__								=			[]
       for token in orig_tokens:
         if token not in never_split:
           if self.do_lower_case:
             lowerCAmelCase__								=			token.lower()
             if self.strip_accents is not False:
               lowerCAmelCase__								=			self._run_strip_accents(__lowerCamelCase	)
           elif self.strip_accents:
             lowerCAmelCase__								=			self._run_strip_accents(__lowerCamelCase	)
         split_tokens.extend(self._run_split_on_punc(__lowerCamelCase					,	__lowerCamelCase	)	)

       lowerCAmelCase__								=			whitespace_tokenize(''' '''.join(__lowerCamelCase	)	)
       return output_tokens
     def 					A__			(	self  :		str					,	__lowerCamelCase  :		str	):



       """simple docstring"""
       lowerCAmelCase__								=			unicodedata.normalize('''NFD'''					,	__lowerCamelCase	)
       lowerCAmelCase__								=			[]
       for char in text:
         lowerCAmelCase__								=			unicodedata.category(__lowerCamelCase	)
         if cat == "Mn":
           continue
         output.append(__lowerCamelCase	)
       return "".join(__lowerCamelCase	)
     def 					A__			(	self  :		Tuple					,	__lowerCamelCase  :		Any					,	__lowerCamelCase  :		str=None	):



       """simple docstring"""
       if not self.do_split_on_punc or (never_split is not None and text in never_split):
         return [text]
       lowerCAmelCase__								=			list(__lowerCamelCase	)
       lowerCAmelCase__								=			0
       lowerCAmelCase__								=			True
       lowerCAmelCase__								=			[]
       while i < len(__lowerCamelCase	):
         lowerCAmelCase__								=			chars[i]
         if _is_punctuation(__lowerCamelCase	):
           output.append([char]	)
           lowerCAmelCase__								=			True
         else:
           if start_new_word:
             output.append([]	)
           lowerCAmelCase__								=			False
           output[-1].append(__lowerCamelCase	)
         i += 1

       return ["".join(__lowerCamelCase	) for x in output]
     def 					A__			(	self  :		str					,	__lowerCamelCase  :		List[str]	):



       """simple docstring"""
       lowerCAmelCase__								=			[]
       for char in text:
         lowerCAmelCase__								=			ord(__lowerCamelCase	)
         if self._is_chinese_char(__lowerCamelCase	):
           output.append(''' '''	)
           output.append(__lowerCamelCase	)
           output.append(''' '''	)
         else:
           output.append(__lowerCamelCase	)
       return "".join(__lowerCamelCase	)
     def 					A__			(	self  :		Optional[Any]					,	__lowerCamelCase  :		Dict	):



       """simple docstring"""
       # This defines a "chinese character" as anything in the CJK Unicode block:
       #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
       #
       # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
       # despite its name. The modern Korean Hangul alphabet is a different block,
       # as is Japanese Hiragana and Katakana. Those alphabets are used to write
       # space-separated words, so they are not treated specially and handled
       # like the all of the other languages.
       if (
           (cp >= 0X4e_00 and cp <= 0X9f_ff)
           or (cp >= 0X34_00 and cp <= 0X4d_bf)  #
           or (cp >= 0X2_00_00 and cp <= 0X2_a6_df)  #
           or (cp >= 0X2_a7_00 and cp <= 0X2_b7_3f)  #
           or (cp >= 0X2_b7_40 and cp <= 0X2_b8_1f)  #
           or (cp >= 0X2_b8_20 and cp <= 0X2_ce_af)  #
           or (cp >= 0Xf9_00 and cp <= 0Xfa_ff)
           or (cp >= 0X2_f8_00 and cp <= 0X2_fa_1f)  #
       ):  #
         return True

       return False



     def 					A__			(	self  :		Dict					,	__lowerCamelCase  :		Dict	):



       """simple docstring"""
       lowerCAmelCase__								=			[]
       for char in text:
         lowerCAmelCase__								=			ord(__lowerCamelCase	)
         if cp == 0 or cp == 0Xff_fd or _is_control(__lowerCamelCase	):
           continue
         if _is_whitespace(__lowerCamelCase	):
           output.append(''' '''	)
         else:
           output.append(__lowerCamelCase	)
       return "".join(__lowerCamelCase	)



def 		a_       (   __lowerCAmelCase							):
  lowerCAmelCase__								=			set()
  lowerCAmelCase__								=			word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)							)
    lowerCAmelCase__								=			char
  return pairs
def 		a_       (   __lowerCAmelCase							):
  lowerCAmelCase__								=			text.replace('''—'''      ,   '''-'''							)
  lowerCAmelCase__								=			text.replace('''–'''      ,   '''-'''							)
  lowerCAmelCase__								=			text.replace('''―'''      ,   '''-'''							)
  lowerCAmelCase__								=			text.replace('''…'''      ,   '''...'''							)
  lowerCAmelCase__								=			text.replace('''´'''      ,   '''\''''							)
  lowerCAmelCase__								=			re.sub(r'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''      ,   r''' \1 '''      ,   __lowerCAmelCase							)
  lowerCAmelCase__								=			re.sub(r'''\s*\n\s*'''      ,   ''' \n '''      ,   __lowerCAmelCase							)
  lowerCAmelCase__								=			re.sub(r'''[^\S\n]+'''      ,   ''' '''      ,   __lowerCAmelCase							)
  return text.strip()





class 						SCREAMING_SNAKE_CASE__		(_a ):
     lowercase_   :			List[str]   =   VOCAB_FILES_NAMES
     lowercase_   :			Dict   =   PRETRAINED_VOCAB_FILES_MAP
     lowercase_   :			Dict   =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowercase_   :			Optional[int]   =   ["input_ids", "attention_mask"]
     def __init__(	self  :		Optional[int]					,	__lowerCamelCase  :		Any					,	__lowerCamelCase  :		Dict					,	__lowerCamelCase  :		Union[str, Any]="<unk>"					,	**__lowerCamelCase  :		List[Any]	):



       """simple docstring"""
       super().__init__(unk_token=__lowerCamelCase					,	**__lowerCamelCase	)

       try:
         import ftfy
         from spacy.lang.en import English

         lowerCAmelCase__								=			English()
         lowerCAmelCase__								=			_nlp.tokenizer
         lowerCAmelCase__								=			ftfy.fix_text
       except ImportError:
         logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''	)
         lowerCAmelCase__								=			BasicTokenizer(do_lower_case=__lowerCamelCase	)
         lowerCAmelCase__								=			None

       with open(__lowerCamelCase					,	encoding='''utf-8'''	) as vocab_handle:
         lowerCAmelCase__								=			json.load(__lowerCamelCase	)
       lowerCAmelCase__								=			{v: k for k, v in self.encoder.items()}
       with open(__lowerCamelCase					,	encoding='''utf-8'''	) as merges_handle:
         lowerCAmelCase__								=			merges_handle.read().split('''\n'''	)[1:-1]
       lowerCAmelCase__								=			[tuple(merge.split()	) for merge in merges]
       lowerCAmelCase__								=			dict(zip(__lowerCamelCase					,	range(len(__lowerCamelCase	)	)	)	)
       lowerCAmelCase__								=			{}
     @property
     def 					A__			(	self  :		List[str]	):



       """simple docstring"""
       return True
     @property
     def 					A__			(	self  :		Union[str, Any]	):



       """simple docstring"""
       return len(self.encoder	)
     def 					A__			(	self  :		Any	):



       """simple docstring"""
       return dict(self.encoder					,	**self.added_tokens_encoder	)
     def 					A__			(	self  :		Optional[Any]					,	__lowerCamelCase  :		List[str]	):



       """simple docstring"""
       lowerCAmelCase__								=			tuple(token[:-1]	) + (token[-1] + '''</w>''',)
       if token in self.cache:
         return self.cache[token]
       lowerCAmelCase__								=			get_pairs(__lowerCamelCase	)

       if not pairs:
         return token + "</w>"

       while True:
         lowerCAmelCase__								=			min(__lowerCamelCase					,	key=lambda __lowerCamelCase	: self.bpe_ranks.get(__lowerCamelCase					,	float('''inf'''	)	)	)
         if bigram not in self.bpe_ranks:
           break
         lowerCAmelCase__							,			lowerCAmelCase__								=			bigram
         lowerCAmelCase__								=			[]
         lowerCAmelCase__								=			0
         while i < len(__lowerCamelCase	):
           try:
             lowerCAmelCase__								=			word.index(__lowerCamelCase					,	__lowerCamelCase	)
           except ValueError:
             new_word.extend(word[i:]	)
             break
           else:
             new_word.extend(word[i:j]	)
             lowerCAmelCase__								=			j

           if word[i] == first and i < len(__lowerCamelCase	) - 1 and word[i + 1] == second:
             new_word.append(first + second	)
             i += 2
           else:
             new_word.append(word[i]	)
             i += 1
         lowerCAmelCase__								=			tuple(__lowerCamelCase	)
         lowerCAmelCase__								=			new_word
         if len(__lowerCamelCase	) == 1:
           break
         else:
           lowerCAmelCase__								=			get_pairs(__lowerCamelCase	)
       lowerCAmelCase__								=			''' '''.join(__lowerCamelCase	)
       if word == "\n  </w>":
         lowerCAmelCase__								=			'''\n</w>'''
       lowerCAmelCase__								=			word
       return word
     def 					A__			(	self  :		List[Any]					,	__lowerCamelCase  :		str	):



       """simple docstring"""
       lowerCAmelCase__								=			[]
       if self.fix_text is None:
         # Using BERT's BasicTokenizer
         lowerCAmelCase__								=			self.nlp.tokenize(__lowerCamelCase	)
         for token in text:
           split_tokens.extend(list(self.bpe(__lowerCamelCase	).split(''' '''	)	)	)
       else:
         # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
         lowerCAmelCase__								=			self.nlp(text_standardize(self.fix_text(__lowerCamelCase	)	)	)
         for token in text:
           split_tokens.extend(list(self.bpe(token.text.lower()	).split(''' '''	)	)	)
       return split_tokens
     def 					A__			(	self  :		List[Any]					,	__lowerCamelCase  :		int	):



       """simple docstring"""
       return self.encoder.get(__lowerCamelCase					,	self.encoder.get(self.unk_token	)	)
     def 					A__			(	self  :		int					,	__lowerCamelCase  :		Any	):



       """simple docstring"""
       return self.decoder.get(__lowerCamelCase					,	self.unk_token	)
     def 					A__			(	self  :		str					,	__lowerCamelCase  :		Any	):



       """simple docstring"""
       lowerCAmelCase__								=			''''''.join(__lowerCamelCase	).replace('''</w>'''					,	''' '''	).strip()
       return out_string



     def 					A__			(	self  :		Optional[int]					,	__lowerCamelCase  :		str					,	__lowerCamelCase  :		Optional[str] = None	):



       """simple docstring"""
       if not os.path.isdir(__lowerCamelCase	):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""	)
         return
       lowerCAmelCase__								=			os.path.join(
           __lowerCamelCase					,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)
       lowerCAmelCase__								=			os.path.join(
           __lowerCamelCase					,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']	)

       with open(__lowerCamelCase					,	'''w'''					,	encoding='''utf-8'''	) as f:
         f.write(json.dumps(self.encoder					,	indent=2					,	sort_keys=__lowerCamelCase					,	ensure_ascii=__lowerCamelCase	) + '''\n'''	)

       lowerCAmelCase__								=			0
       with open(__lowerCamelCase					,	'''w'''					,	encoding='''utf-8'''	) as writer:
         writer.write('''#version: 0.2\n'''	)
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,	key=lambda __lowerCamelCase	: kv[1]	):
           if index != token_index:
             logger.warning(
                 F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                 ''' Please check that the tokenizer is not corrupted!'''	)
             lowerCAmelCase__								=			token_index
           writer.write(''' '''.join(__lowerCamelCase	) + '''\n'''	)
           index += 1

       return vocab_file, merge_file







import os
from dataclasses import replace

import jax
import wandb
from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
from datasets import load_dataset
from flax import jax_utils

from transformers import BigBirdTokenizerFast


if __name__ == "__main__":
      print("""#################### AVAILABLE DEVICES ####################""")
      print(jax.devices())
      print("""###########################################################""")

      # setup for wandb sweep
      __magic_name__     :     Optional[Any]  							=					Args()
      __magic_name__     :     List[Any]  							=					wandb.init(project="""bigbird-natural-questions""", config=args.__dict__)
      __magic_name__     :     Any  							=					dict(logger.config)
      del wandb_args["batch_size"]
      __magic_name__     :     int  							=					replace(args, **wandb_args)
      __magic_name__     :     Tuple  							=					args.base_dir + """-""" + wandb.run.id
      __magic_name__     :     Union[str, Any]  							=					replace(args, base_dir=base_dir)
      print(args)

      __magic_name__     :     Union[str, Any]  							=					load_dataset("""json""", data_files=args.tr_data_path)["""train"""]
      __magic_name__     :     List[Any]  							=					load_dataset("""json""", data_files=args.val_data_path)["""train"""]

      # drop extra batch for now
      __magic_name__     :     List[str]  							=					range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
      __magic_name__     :     Dict  							=					tr_dataset.shuffle().select(indices)
      __magic_name__     :     Any  							=					range(len(val_dataset) - len(val_dataset) % args.batch_size)
      __magic_name__     :     Dict  							=					val_dataset.shuffle().select(indices)

      if os.environ.get("""TRAIN_ON_SMALL""", """false""") == "true":
            __magic_name__     :     Any  							=					tr_dataset.shuffle().select(range(8_00_00))
            __magic_name__     :     Optional[int]  							=					val_dataset.shuffle().select(range(80_00))

      print(tr_dataset)
      print(val_dataset)

      __magic_name__     :     Union[str, Any]  							=					FlaxBigBirdForNaturalQuestions.from_pretrained(
          args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
      )
      __magic_name__     :     Tuple  							=					BigBirdTokenizerFast.from_pretrained(args.model_id)
      __magic_name__     :     Tuple  							=					DataCollator(pad_id=tokenizer.pad_token_id, max_length=40_96)

      __magic_name__     :     int  							=					{
          """lr""": args.lr,
          """init_lr""": args.init_lr,
          """warmup_steps""": args.warmup_steps,
          """num_train_steps""": args.max_epochs * (len(tr_dataset) // args.batch_size),
          """weight_decay""": args.weight_decay,
      }
      __magic_name__							,   __magic_name__     :     Optional[int]  							=					build_tx(**tx_args)

      __magic_name__     :     List[str]  							=					Trainer(
          args=args,
          data_collator=data_collator,
          model_save_fn=model.save_pretrained,
          train_step_fn=train_step,
          val_step_fn=val_step,
          logger=logger,
          scheduler_fn=lr,
      )

      __magic_name__     :     Union[str, Any]  							=					None
      __magic_name__     :     Dict  							=					trainer.create_state(model, tx, num_train_steps=tx_args["""num_train_steps"""], ckpt_dir=ckpt_dir)
      try:
            trainer.train(state, tr_dataset, val_dataset)
      except KeyboardInterrupt:
            print("""Oooops; TRAINING STOPPED UNFORTUNATELY""")

      print("""SAVING WEIGHTS IN `final-weights`""")
      __magic_name__     :     List[Any]  							=					jax_utils.unreplicate(state.params)
      model.save_pretrained(os.path.join(args.base_dir, """final-weights"""), params=params)

