

"""simple docstring"""





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
   if is_torch_available():
      import torch
   if is_tf_available():
      import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_a				   =							logging.get_logger(__name__)

_a				   =							{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_a				   =							{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

_a				   =							{
    'Salesforce/codegen-350M-mono': 20_48,
}




@lru_cache()
def 						_A						(						)			->		Dict:
   '''simple docstring'''


   __lowercase			=		(
       list(range(ord("!"),    ord("~") + 1)) + list(range(ord("¡"),    ord("¬") + 1)) + list(range(ord("®"),    ord("ÿ") + 1))
   )
   __lowercase			=		bs[:]
   __lowercase			=		0
   for b in range(2**8):
      if b not in bs:
         bs.append(__a)
         cs.append(2**8 + n)
         n += 1
   __lowercase			=		[chr(__a) for n in cs]
   return dict(zip(__a,    __a))




def 						_A						(						UpperCamelCase_  : List[Any])			->		Tuple:
   '''simple docstring'''


   __lowercase			=		set()
   __lowercase			=		word[0]
   for char in word[1:]:
      pairs.add((prev_char, char))
      __lowercase			=		char
   return pairs




class       _lowerCAmelCase						(			UpperCamelCase__     ):







 """simple docstring"""







 __UpperCAmelCase			:      Optional[int]     	=			VOCAB_FILES_NAMES
 __UpperCAmelCase			:      Optional[Any]     	=			PRETRAINED_VOCAB_FILES_MAP
 __UpperCAmelCase			:      Optional[Any]     	=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __UpperCAmelCase			:      Dict     	=			["input_ids", "attention_mask"]


 def __init__(    self							:    Any,      UpperCAmelCase__							:    Union[str, Any],      UpperCAmelCase__							:    Dict,      UpperCAmelCase__							:    Union[str, Any]="replace",      UpperCAmelCase__							:    Dict="<|endoftext|>",      UpperCAmelCase__							:    List[Any]="<|endoftext|>",      UpperCAmelCase__							:    Dict="<|endoftext|>",      UpperCAmelCase__							:    List[Any]=None,      UpperCAmelCase__							:    int=False,      UpperCAmelCase__							:    int=False,      **UpperCAmelCase__							:    Any,      ):
    __lowercase			=		AddedToken(snake_case_,      lstrip=snake_case_,      rstrip=snake_case_			) if isinstance(snake_case_,      snake_case_			) else bos_token
    __lowercase			=		AddedToken(snake_case_,      lstrip=snake_case_,      rstrip=snake_case_			) if isinstance(snake_case_,      snake_case_			) else eos_token
    __lowercase			=		AddedToken(snake_case_,      lstrip=snake_case_,      rstrip=snake_case_			) if isinstance(snake_case_,      snake_case_			) else unk_token
    __lowercase			=		AddedToken(snake_case_,      lstrip=snake_case_,      rstrip=snake_case_			) if isinstance(snake_case_,      snake_case_			) else pad_token
    super().__init__(
        errors=snake_case_,      unk_token=snake_case_,      bos_token=snake_case_,      eos_token=snake_case_,      pad_token=snake_case_,      add_prefix_space=snake_case_,      add_bos_token=snake_case_,      **snake_case_,      )
    __lowercase			=		add_bos_token

    with open(snake_case_,      encoding="utf-8"			) as vocab_handle:
       __lowercase			=		json.load(snake_case_			)
    __lowercase			=		{v: k for k, v in self.encoder.items()}
    __lowercase			=		errors  # how to handle errors in decoding
    __lowercase			=		bytes_to_unicode()
    __lowercase			=		{v: k for k, v in self.byte_encoder.items()}
    with open(snake_case_,      encoding="utf-8"			) as merges_handle:
       __lowercase			=		merges_handle.read().split("\n"			)[1:-1]
    __lowercase			=		[tuple(merge.split()			) for merge in bpe_merges]
    __lowercase			=		dict(zip(snake_case_,      range(len(snake_case_			)			)			)			)
    __lowercase			=		{}
    __lowercase			=		add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    __lowercase			=		re.compile(r"\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"			)


 @property
 def       _lowercase				(    self							:    Union[str, Any]			):
    return len(self.encoder			)


 def       _lowercase				(    self							:    Optional[Any]			):
    return dict(self.encoder,      **self.added_tokens_encoder			)


 def       _lowercase				(    self							:    str,      UpperCAmelCase__							:    Optional[int]			):
    if token in self.cache:
       return self.cache[token]
    __lowercase			=		tuple(snake_case_			)
    __lowercase			=		get_pairs(snake_case_			)

    if not pairs:
       return token

    while True:
       __lowercase			=		min(snake_case_,      key=lambda UpperCAmelCase__			: self.bpe_ranks.get(snake_case_,      float("inf"			)			)			)
       if bigram not in self.bpe_ranks:
          break
       __lowercase			=		bigram
       __lowercase			=		[]
       __lowercase			=		0
       while i < len(snake_case_			):
          try:
             __lowercase			=		word.index(snake_case_,      snake_case_			)
          except ValueError:
             new_word.extend(word[i:]			)
             break
          else:
             new_word.extend(word[i:j]			)
             __lowercase			=		j

          if word[i] == first and i < len(snake_case_			) - 1 and word[i + 1] == second:
             new_word.append(first + second			)
             i += 2
          else:
             new_word.append(word[i]			)
             i += 1
       __lowercase			=		tuple(snake_case_			)
       __lowercase			=		new_word
       if len(snake_case_			) == 1:
          break
       else:
          __lowercase			=		get_pairs(snake_case_			)
    __lowercase			=		''' '''.join(snake_case_			)
    __lowercase			=		word
    return word


 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    int,      UpperCAmelCase__							:    List[str]=None			):
    if self.add_bos_token:
       __lowercase			=		[self.bos_token_id]
    else:
       __lowercase			=		[]

    __lowercase			=		bos_token_ids + token_ids_a

    if token_ids_a is None:
       return output

    return output + bos_token_ids + token_ids_a


 def       _lowercase				(    self							:    Tuple,      UpperCAmelCase__							:    Union[str, Any]			):
    __lowercase			=		[]
    for token in re.findall(self.pat,      snake_case_			):
       __lowercase			=		''''''.join(
           self.byte_encoder[b] for b in token.encode("utf-8"			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case_			).split(" "			)			)
    return bpe_tokens


 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    List[str]			):
    return self.encoder.get(snake_case_,      self.encoder.get(self.unk_token			)			)


 def       _lowercase				(    self							:    Optional[int],      UpperCAmelCase__							:    Optional[Any]			):
    return self.decoder.get(snake_case_			)


 def       _lowercase				(    self							:    List[Any],      UpperCAmelCase__							:    Dict			):
    __lowercase			=		''''''.join(snake_case_			)
    __lowercase			=		bytearray([self.byte_decoder[c] for c in text]			).decode("utf-8",      errors=self.errors			)
    return text


 def       _lowercase				(    self							:    Optional[Any],      UpperCAmelCase__							:    Tuple,      UpperCAmelCase__							:    List[Any] = None			):
    if not os.path.isdir(snake_case_			):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""			)
       return
    __lowercase			=		os.path.join(
        snake_case_,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
    __lowercase			=		os.path.join(
        snake_case_,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

    with open(snake_case_,      "w",      encoding="utf-8"			) as f:
       f.write(json.dumps(self.encoder,      indent=2,      sort_keys=snake_case_,      ensure_ascii=snake_case_			) + "\n"			)

    __lowercase			=		0
    with open(snake_case_,      "w",      encoding="utf-8"			) as writer:
       writer.write("#version: 0.2\n"			)
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),      key=lambda UpperCAmelCase__			: kv[1]			):
          if index != token_index:
             logger.warning(
                 F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                 " Please check that the tokenizer is not corrupted!"			)
             __lowercase			=		token_index
          writer.write(" ".join(snake_case_			) + "\n"			)
          index += 1

    return vocab_file, merge_file


 def       _lowercase				(    self							:    Any,      UpperCAmelCase__							:    Optional[Any],      UpperCAmelCase__							:    List[Any]=False,      **UpperCAmelCase__							:    List[str]			):
    __lowercase			=		kwargs.pop("add_prefix_space",      self.add_prefix_space			)
    if is_split_into_words or add_prefix_space:
       __lowercase			=		''' ''' + text
    return (text, kwargs)


 def       _lowercase				(    self							:    str,      UpperCAmelCase__							:    Tuple,      UpperCAmelCase__							:    Union[str, Any] = False,      UpperCAmelCase__							:    Union[str, Any] = None,      UpperCAmelCase__							:    List[str] = None,      **UpperCAmelCase__							:    Dict,      ):
    __lowercase			=		super()._decode(
        token_ids=snake_case_,      skip_special_tokens=snake_case_,      clean_up_tokenization_spaces=snake_case_,      **snake_case_,      )

    if truncate_before_pattern is not None and len(snake_case_			) > 0:
       __lowercase			=		self.truncate(snake_case_,      snake_case_			)

    return decoded_text



 def       _lowercase				(    self							:    Optional[Any],      UpperCAmelCase__							:    Optional[Any],      UpperCAmelCase__							:    Union[str, Any]			):
    def find_re(UpperCAmelCase__							:    Any,      UpperCAmelCase__							:    int,      UpperCAmelCase__							:    Optional[int]			):
       __lowercase			=		pattern.search(snake_case_,      snake_case_			)
       return m.start() if m else -1

    __lowercase			=		[re.compile(snake_case_,      re.MULTILINE			) for pattern in truncate_before_pattern]

    __lowercase			=		list(re.finditer("^print",      snake_case_,      re.MULTILINE			)			)

    if len(snake_case_			) > 1:
       __lowercase			=		completion[: prints[1].start()]

    __lowercase			=		list(re.finditer("^def",      snake_case_,      re.MULTILINE			)			)

    if len(snake_case_			) > 1:
       __lowercase			=		completion[: defs[1].start()]

    __lowercase			=		0

    __lowercase			=		[
        pos for pos in [find_re(snake_case_,      snake_case_,      snake_case_			) for terminal in terminals] if pos != -1
    ]

    if len(snake_case_			) > 0:
       return completion[: min(snake_case_			)]
    else:
       return completion







"""simple docstring"""





import os
from dataclasses import replace

import jax
import wandb
from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
from datasets import load_dataset
from flax import jax_utils

from transformers import BigBirdTokenizerFast


if __name__ == "__main__":
   print('#################### AVAILABLE DEVICES ####################')
   print(jax.devices())
   print('###########################################################')

   # setup for wandb sweep
   _a				   =							Args()
   _a				   =							wandb.init(project='bigbird-natural-questions', config=args.__dict__)
   _a				   =							dict(logger.config)
   del wandb_args["batch_size"]
   _a				   =							replace(args, **wandb_args)
   _a				   =							args.base_dir + '-' + wandb.run.id
   _a				   =							replace(args, base_dir=base_dir)
   print(args)

   _a				   =							load_dataset('json', data_files=args.tr_data_path)['train']
   _a				   =							load_dataset('json', data_files=args.val_data_path)['train']

   # drop extra batch for now
   _a				   =							range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
   _a				   =							tr_dataset.shuffle().select(indices)
   _a				   =							range(len(val_dataset) - len(val_dataset) % args.batch_size)
   _a				   =							val_dataset.shuffle().select(indices)

   if os.environ.get('TRAIN_ON_SMALL', 'false') == "true":
      _a				   =							tr_dataset.shuffle().select(range(8_00_00))
      _a				   =							val_dataset.shuffle().select(range(80_00))

   print(tr_dataset)
   print(val_dataset)

   _a				   =							FlaxBigBirdForNaturalQuestions.from_pretrained(
       args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
   )
   _a				   =							BigBirdTokenizerFast.from_pretrained(args.model_id)
   _a				   =							DataCollator(pad_id=tokenizer.pad_token_id, max_length=40_96)

   _a				   =							{
       'lr': args.lr,
       'init_lr': args.init_lr,
       'warmup_steps': args.warmup_steps,
       'num_train_steps': args.max_epochs * (len(tr_dataset) // args.batch_size),
       'weight_decay': args.weight_decay,
   }
   _a   ,		_a				   =							build_tx(**tx_args)

   _a				   =							Trainer(
       args=args,
       data_collator=data_collator,
       model_save_fn=model.save_pretrained,
       train_step_fn=train_step,
       val_step_fn=val_step,
       logger=logger,
       scheduler_fn=lr,
   )

   _a				   =							None
   _a				   =							trainer.create_state(model, tx, num_train_steps=tx_args['num_train_steps'], ckpt_dir=ckpt_dir)
   try:
      trainer.train(state, tr_dataset, val_dataset)
   except KeyboardInterrupt:
      print('Oooops; TRAINING STOPPED UNFORTUNATELY')

   print('SAVING WEIGHTS IN `final-weights`')
   _a				   =							jax_utils.unreplicate(state.params)
   model.save_pretrained(os.path.join(args.base_dir, 'final-weights'), params=params)




