

"""simple docstring"""





import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)




class       _lowerCAmelCase						:







 """simple docstring"""







 def __init__(    self							:    str,      UpperCAmelCase__							:    Dict			):
    __lowercase			=		foo



 def __call__(    self							:    Optional[Any]			):
    return self.foo







class       _lowerCAmelCase						(			datasets.Dataset     ):







 """simple docstring"""







 @fingerprint_transform(inplace=UpperCAmelCase__			)
 def       _lowercase				(    self							:    Optional[Any],      UpperCAmelCase__							:    int,      *UpperCAmelCase__							:    List[str],      **UpperCAmelCase__							:    List[str]			):
    return DatasetChild(self.data,      fingerprint=UpperCAmelCase__			)



 @fingerprint_transform(inplace=UpperCAmelCase__			)
 def       _lowercase				(    self							:    str,      UpperCAmelCase__							:    List[str],      *UpperCAmelCase__							:    int,      **UpperCAmelCase__							:    Optional[int]			):
    return DatasetChild(self.data,      fingerprint=UpperCAmelCase__			)







class       _lowerCAmelCase						:







 """simple docstring"""







 def __init__(    self							:    Dict,      UpperCAmelCase__							:    List[Any]			):
    __lowercase			=		callable


 def __call__(    self							:    Optional[Any],      *UpperCAmelCase__							:    Union[str, Any],      **UpperCAmelCase__							:    Any			):
    if self.callable is not None:
       return self.callable(*UpperCAmelCase__,      **UpperCAmelCase__			)



 def __getstate__(    self							:    Any			):
    raise pickle.PicklingError()







class       _lowerCAmelCase						(			lowercase     ):







 """simple docstring"""







 @require_transformers
 @pytest.mark.integration
 def       _lowercase				(    self							:    List[Any]			):
    from transformers import AutoTokenizer

    def encode(UpperCAmelCase__							:    Any			):
       return tokenizer(UpperCAmelCase__			)

    # TODO: add hash consistency tests across sessions
    __lowercase			=		AutoTokenizer.from_pretrained("bert-base-uncased"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase__			: tokenizer(UpperCAmelCase__			)			)			).hexdigest()
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		AutoTokenizer.from_pretrained("bert-base-cased"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase__			: tokenizer(UpperCAmelCase__			)			)			).hexdigest()
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		AutoTokenizer.from_pretrained("bert-base-uncased"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase__			: tokenizer(UpperCAmelCase__			)			)			).hexdigest()
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)


 @require_transformers
 @pytest.mark.integration
 def       _lowercase				(    self							:    str			):
    from transformers import AutoTokenizer

    __lowercase			=		AutoTokenizer.from_pretrained("gpt2"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    tokenizer("Hello world !"			)  # call once to change the tokenizer's cache
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)



 @require_regex
 def       _lowercase				(    self							:    Optional[int]			):
    import regex

    __lowercase			=		regex.Regex("foo"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		regex.Regex("bar"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		regex.Regex("foo"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)







class       _lowerCAmelCase						(			lowercase     ):







 """simple docstring"""







 def       _lowercase				(    self							:    Optional[Any]			):
    def func():
       return foo

    __lowercase			=		[0]
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		[1]
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		[0]
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)


 def       _lowercase				(    self							:    Union[str, Any]			):
    def func():
       pass

    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()

    def func():
       pass

    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)


 def       _lowercase				(    self							:    str			):
    __lowercase			=		mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
    __lowercase			=		mda(datasets.utils.py_utils.dumps(Foo([1]			)			)			).hexdigest()
    __lowercase			=		mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)


 def       _lowercase				(    self							:    int			):
    __lowercase			=		mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
    __lowercase			=		mda(datasets.utils.py_utils.dumps(Foo([1]			).__call__			)			).hexdigest()
    __lowercase			=		mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)


 def       _lowercase				(    self							:    List[Any]			):
    __lowercase			=		(
        "co_argcount",
        "co_kwonlyargcount",
        "co_nlocals",
        "co_stacksize",
        "co_flags",
        "co_code",
        "co_consts",
        "co_names",
        "co_varnames",
        "co_filename",
        "co_name",
        "co_firstlineno",
        "co_lnotab",
        "co_freevars",
        "co_cellvars",
    )

    def _create_code_pyaa(*UpperCAmelCase__							:    List[str]			):
       if hasattr(args[-3],      "encode"			):
          __lowercase			=		list(UpperCAmelCase__			)
          __lowercase			=		args[-3].encode()  # co_lnotab
          __lowercase			=		args[-1_0].encode()  # co_code
       if hasattr(UpperCAmelCase__,      "co_posonlyargcount"			):
          if len(UpperCAmelCase__			) == 1_6:
             return CodeType(*UpperCAmelCase__			)
          elif len(UpperCAmelCase__			) == 1_5:
             return CodeType(args[0],      0,      *args[1:]			)
          return CodeType(args[0],      0,      0,      *args[1:]			)
       elif hasattr(UpperCAmelCase__,      "co_kwonlyargcount"			):
          if len(UpperCAmelCase__			) == 1_6:
             return CodeType(args[0],      *args[2:]			)
          elif len(UpperCAmelCase__			) == 1_5:
             return CodeType(*UpperCAmelCase__			)
          return CodeType(args[0],      0,      *args[1:]			)
       if len(UpperCAmelCase__			) == 1_6:
          return CodeType(args[0],      *args[3:]			)
       elif len(UpperCAmelCase__			) == 1_5:
          return CodeType(args[0],      *args[2:]			)
       return CodeType(*UpperCAmelCase__			)

    def create_ipython_func(UpperCAmelCase__							:    str,      UpperCAmelCase__							:    Union[str, Any]			):
       def func():
          return returned_obj

       __lowercase			=		func.__code__
       # Use _create_code from dill in order to make it work for different python versions
       if datasets.config.PY_VERSION < version.parse("3.8"			):
          __lowercase			=		_create_code_pyaa(
              *[getattr(UpperCAmelCase__,      UpperCAmelCase__			) if k != "co_filename" else co_filename for k in code_args_pyaa]			)
       else:
          __lowercase			=		code.replace(co_filename=UpperCAmelCase__			)
       return FunctionType(UpperCAmelCase__,      func.__globals__,      func.__name__,      func.__defaults__,      func.__closure__			)

    __lowercase	,__lowercase			=		"<ipython-input-2-e0383a102aae>", [0]
    __lowercase			=		mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__,      UpperCAmelCase__			)			)			).hexdigest()
    __lowercase	,__lowercase			=		"<ipython-input-2-e0383a102aae>", [1]
    __lowercase			=		mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__,      UpperCAmelCase__			)			)			).hexdigest()
    __lowercase	,__lowercase			=		"<ipython-input-5-713f6613acf3>", [0]
    __lowercase			=		mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__,      UpperCAmelCase__			)			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)

    __lowercase	,__lowercase			=		os.path.join(gettempdir(),      "ipykernel_12345",      "321456789.py"			), [0]
    __lowercase			=		mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__,      UpperCAmelCase__			)			)			).hexdigest()
    __lowercase	,__lowercase			=		os.path.join(gettempdir(),      "ipykernel_12345",      "321456789.py"			), [1]
    __lowercase			=		mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__,      UpperCAmelCase__			)			)			).hexdigest()
    __lowercase	,__lowercase			=		os.path.join(gettempdir(),      "ipykernel_12345",      "654123987.py"			), [0]
    __lowercase			=		mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase__,      UpperCAmelCase__			)			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)



 def       _lowercase				(    self							:    Optional[Any]			):
    __lowercase	,__lowercase			=		[0], [1]

    def func():
       return foo, bar

    __lowercase			=		"__main__"

    def globalvars_mocka_side_effect(UpperCAmelCase__							:    str,      *UpperCAmelCase__							:    Tuple,      **UpperCAmelCase__							:    Tuple			):
       return {"foo": foo, "bar": bar}

    def globalvars_mocka_side_effect(UpperCAmelCase__							:    Union[str, Any],      *UpperCAmelCase__							:    Union[str, Any],      **UpperCAmelCase__							:    Any			):
       return {"bar": bar, "foo": foo}

    with patch("dill.detect.globalvars",      side_effect=UpperCAmelCase__			) as globalvars_mocka:
       __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
       self.assertGreater(globalvars_mocka.call_count,      0			)
    with patch("dill.detect.globalvars",      side_effect=UpperCAmelCase__			) as globalvars_mocka:
       __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
       self.assertGreater(globalvars_mocka.call_count,      0			)
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)







class       _lowerCAmelCase						(			lowercase     ):







 """simple docstring"""







 def       _lowercase				(    self							:    Optional[Any]			):
    __lowercase			=		Hasher.hash("hello"			)
    __lowercase			=		Hasher.hash("hello"			)
    __lowercase			=		Hasher.hash("there"			)
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)


 def       _lowercase				(    self							:    Optional[int]			):
    __lowercase			=		Hasher.hash(Foo("hello"			)			)
    __lowercase			=		Hasher.hash(Foo("hello"			)			)
    __lowercase			=		Hasher.hash(Foo("there"			)			)
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)


 def       _lowercase				(    self							:    List[str]			):
    __lowercase			=		Hasher()
    for x in ["hello", Foo("hello"			)]:
       hasher.update(UpperCAmelCase__			)
    __lowercase			=		hasher.hexdigest()
    __lowercase			=		Hasher()
    for x in ["hello", Foo("hello"			)]:
       hasher.update(UpperCAmelCase__			)
    __lowercase			=		hasher.hexdigest()
    __lowercase			=		Hasher()
    for x in ["there", Foo("there"			)]:
       hasher.update(UpperCAmelCase__			)
    __lowercase			=		hasher.hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)


 def       _lowercase				(    self							:    Optional[Any]			):
    with self.assertRaises(pickle.PicklingError			):
       Hasher.hash(UnpicklableCallable(Foo("hello"			)			)			)


 def       _lowercase				(    self							:    List[str]			):
    __lowercase			=		"abc"
    __lowercase			=		[string, string]  # two strings have the same ids
    __lowercase			=		[string, string]
    __lowercase			=		json.loads(F"""[\"{string}\", \"{string}\"]"""			)  # two strings have different ids
    self.assertIs(obja[0],      UpperCAmelCase__			)
    self.assertIs(obja[0],      obja[1]			)
    self.assertIs(obja[0],      UpperCAmelCase__			)
    self.assertIs(obja[0],      obja[1]			)
    self.assertIsNot(obja[0],      UpperCAmelCase__			)
    self.assertIsNot(obja[0],      obja[1]			)
    __lowercase			=		Hasher.hash(UpperCAmelCase__			)
    __lowercase			=		Hasher.hash(UpperCAmelCase__			)
    __lowercase			=		Hasher.hash(UpperCAmelCase__			)
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)


 @require_tiktoken
 def       _lowercase				(    self							:    Tuple			):
    import tiktoken

    __lowercase			=		tiktoken.get_encoding("gpt2"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		tiktoken.get_encoding("r50k_base"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		tiktoken.get_encoding("gpt2"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)


 @require_torch
 def       _lowercase				(    self							:    List[Any]			):
    import torch

    __lowercase			=		torch.tensor([1.0]			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		torch.tensor([2.0]			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		torch.tensor([1.0]			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)



 @require_spacy
 @require_spacy_model("en_core_web_sm"			)
 @require_spacy_model("fr_core_news_sm"			)
 @pytest.mark.integration
 def       _lowercase				(    self							:    Tuple			):
    import spacy

    __lowercase			=		spacy.load("en_core_web_sm"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		spacy.load("fr_core_news_sm"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    __lowercase			=		spacy.load("en_core_web_sm"			)
    __lowercase			=		mda(datasets.utils.py_utils.dumps(UpperCAmelCase__			)			).hexdigest()
    self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)
    self.assertNotEqual(UpperCAmelCase__,      UpperCAmelCase__			)







@pytest.mark.integration
def 						_A						(						UpperCamelCase_  : Path)			->		Dict:
   '''simple docstring'''


   __lowercase			=		tmp_path / "dir1"
   __lowercase			=		tmp_path / "dir2"
   dira.mkdir()
   dira.mkdir()
   __lowercase			=		"script.py"
   __lowercase			=		dedent(
       "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    ")
   __lowercase			=		dira / script_filename
   __lowercase			=		dira / script_filename
   with script_patha.open("w") as f:
      f.write(UpperCamelCase_)
   with script_patha.open("w") as f:
      f.write(UpperCamelCase_)
   __lowercase			=		subprocess.check_output(["python", str(UpperCamelCase_)])
   __lowercase			=		subprocess.check_output(["python", str(UpperCamelCase_)])
   assert fingerprinta == fingerprinta




def 						_A						(						)			->		Optional[Any]:
   '''simple docstring'''


   __lowercase			=		{"a": [0, 1, 2]}
   __lowercase			=		DatasetChild(InMemoryTable.from_pydict(UpperCamelCase_))
   __lowercase			=		dataset.funca()._fingerprint
   assert expected_fingerprint == dataset.funca()._fingerprint
   assert expected_fingerprint != dataset.funca()._fingerprint

   with Pool(2) as p:
      assert expected_fingerprint == p.apply_async(dataset.funca).get()._fingerprint
      assert expected_fingerprint != p.apply_async(dataset.funca).get()._fingerprint




def 						_A						(						)			->		List[Any]:
   '''simple docstring'''


   __lowercase			=		{"a": [0, 1, 2]}




   class       _lowerCAmelCase						(			datasets.Dataset     ):







    """simple docstring"""







    @fingerprint_transform(inplace=UpperCAmelCase__			)
    def       _lowercase				(    self							:    Any,      UpperCAmelCase__							:    List[str]			):
       return DummyDatasetChild(self.data,      fingerprint=UpperCAmelCase__			)







   __lowercase			=		DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase_)).func()




   class       _lowerCAmelCase						(			datasets.Dataset     ):







    """simple docstring"""







    @fingerprint_transform(inplace=UpperCAmelCase__,      version="1.0.0"			)
    def       _lowercase				(    self							:    List[str],      UpperCAmelCase__							:    Union[str, Any]			):
       return DummyDatasetChild(self.data,      fingerprint=UpperCAmelCase__			)







   __lowercase			=		DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase_)).func()




   class       _lowerCAmelCase						(			datasets.Dataset     ):







    """simple docstring"""







    @fingerprint_transform(inplace=UpperCAmelCase__,      version="2.0.0"			)
    def       _lowercase				(    self							:    Any,      UpperCAmelCase__							:    List[str]			):
       return DummyDatasetChild(self.data,      fingerprint=UpperCAmelCase__			)







   __lowercase			=		DummyDatasetChild(InMemoryTable.from_pydict(UpperCamelCase_)).func()

   assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}) == 3




def 						_A						(						)			->		str:
   '''simple docstring'''


   __lowercase			=		Hasher()
   hasher.update(lambda UpperCamelCase_: x)







"""simple docstring"""





import os
from dataclasses import replace

import jax
import wandb
from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
from datasets import load_dataset
from flax import jax_utils

from transformers import BigBirdTokenizerFast


if __name__ == "__main__":
   print('#################### AVAILABLE DEVICES ####################')
   print(jax.devices())
   print('###########################################################')

   # setup for wandb sweep
   _a				   =							Args()
   _a				   =							wandb.init(project='bigbird-natural-questions', config=args.__dict__)
   _a				   =							dict(logger.config)
   del wandb_args["batch_size"]
   _a				   =							replace(args, **wandb_args)
   _a				   =							args.base_dir + '-' + wandb.run.id
   _a				   =							replace(args, base_dir=base_dir)
   print(args)

   _a				   =							load_dataset('json', data_files=args.tr_data_path)['train']
   _a				   =							load_dataset('json', data_files=args.val_data_path)['train']

   # drop extra batch for now
   _a				   =							range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
   _a				   =							tr_dataset.shuffle().select(indices)
   _a				   =							range(len(val_dataset) - len(val_dataset) % args.batch_size)
   _a				   =							val_dataset.shuffle().select(indices)

   if os.environ.get('TRAIN_ON_SMALL', 'false') == "true":
      _a				   =							tr_dataset.shuffle().select(range(8_00_00))
      _a				   =							val_dataset.shuffle().select(range(80_00))

   print(tr_dataset)
   print(val_dataset)

   _a				   =							FlaxBigBirdForNaturalQuestions.from_pretrained(
       args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
   )
   _a				   =							BigBirdTokenizerFast.from_pretrained(args.model_id)
   _a				   =							DataCollator(pad_id=tokenizer.pad_token_id, max_length=40_96)

   _a				   =							{
       'lr': args.lr,
       'init_lr': args.init_lr,
       'warmup_steps': args.warmup_steps,
       'num_train_steps': args.max_epochs * (len(tr_dataset) // args.batch_size),
       'weight_decay': args.weight_decay,
   }
   _a   ,		_a				   =							build_tx(**tx_args)

   _a				   =							Trainer(
       args=args,
       data_collator=data_collator,
       model_save_fn=model.save_pretrained,
       train_step_fn=train_step,
       val_step_fn=val_step,
       logger=logger,
       scheduler_fn=lr,
   )

   _a				   =							None
   _a				   =							trainer.create_state(model, tx, num_train_steps=tx_args['num_train_steps'], ckpt_dir=ckpt_dir)
   try:
      trainer.train(state, tr_dataset, val_dataset)
   except KeyboardInterrupt:
      print('Oooops; TRAINING STOPPED UNFORTUNATELY')

   print('SAVING WEIGHTS IN `final-weights`')
   _a				   =							jax_utils.unreplicate(state.params)
   model.save_pretrained(os.path.join(args.base_dir, 'final-weights'), params=params)




