




'''simple docstring'''





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
   from tokenizers import Tokenizer
   from tokenizers.models import WordPiece

class      _a       (		unittest.TestCase							):






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE					)							->    int:
          _snake_case							=   list(tokenizer_class.max_model_input_sizes.keys()					)
          for model_name in sa_models[:1]:
               _snake_case							=   tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE					)
               self.assertIsNotNone(_SCREAMING_SNAKE_CASE					)
               self.assertIsInstance(_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE					)
               self.assertIsInstance(_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE					)

               for special_tok in tokenizer.all_special_tokens:
                    self.assertIsInstance(_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE					)
                    _snake_case							=   tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE					)
                    self.assertIsInstance(_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE					)






     def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE = None					)							->    str:
          _snake_case							=   pickle.dumps(_SCREAMING_SNAKE_CASE					)
          self.assertIsNotNone(_SCREAMING_SNAKE_CASE					)

          _snake_case							=   pickle.loads(_SCREAMING_SNAKE_CASE					)

          # Ensure is_fast is correctly restored
          self.assertEqual(be_restored.is_fast      ,be_original.is_fast					)

          # Ensure encodings are potentially correctly restored
          if be_original.is_fast:
               self.assertIsNotNone(be_restored.encodings					)
          else:
               self.assertIsNone(be_restored.encodings					)

          # Ensure the keys are the same
          for original_v, restored_v in zip(be_original.values()      ,be_restored.values()					):
               if equal_op:
                    self.assertTrue(equal_op(_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE					)					)
               else:
                    self.assertEqual(_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE					)






     @slow
     def        _lowercase			(				self					)							->    int:
          self.check_tokenizer_from_pretrained(_SCREAMING_SNAKE_CASE					)






     def        _lowercase			(				self					)							->    int:
          self.assertEqual(TensorType("tf"					)      ,TensorType.TENSORFLOW					)
          self.assertEqual(TensorType("pt"					)      ,TensorType.PYTORCH					)
          self.assertEqual(TensorType("np"					)      ,TensorType.NUMPY					)






     @require_tokenizers
     def        _lowercase			(				self					)							->    int:
          import numpy as np

          _snake_case							=   BertTokenizer.from_pretrained("bert-base-cased"					)
          _snake_case							=   BertTokenizerFast.from_pretrained("bert-base-cased"					)

          # Python no tensor
          with self.subTest("BatchEncoding (Python, return_tensors=None)"					):
               self.assert_dump_and_restore(tokenizer_p("Small example to encode"					)					)

          with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"					):
               self.assert_dump_and_restore(
                   tokenizer_p("Small example to encode"      ,return_tensors=TensorType.NUMPY					)      ,np.array_equal					)

          with self.subTest("BatchEncoding (Rust, return_tensors=None)"					):
               self.assert_dump_and_restore(tokenizer_r("Small example to encode"					)					)

          with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"					):
               self.assert_dump_and_restore(
                   tokenizer_r("Small example to encode"      ,return_tensors=TensorType.NUMPY					)      ,np.array_equal					)






     @require_tf
     @require_tokenizers
     def        _lowercase			(				self					)							->    Any:
          import tensorflow as tf

          def tf_array_equals(_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE					):
               return tf.reduce_all(tf.equal(_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE					)					)

          _snake_case							=   BertTokenizer.from_pretrained("bert-base-cased"					)
          _snake_case							=   BertTokenizerFast.from_pretrained("bert-base-cased"					)

          with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"					):
               self.assert_dump_and_restore(
                   tokenizer_p("Small example to encode"      ,return_tensors=TensorType.TENSORFLOW					)      ,_SCREAMING_SNAKE_CASE					)

          with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"					):
               self.assert_dump_and_restore(
                   tokenizer_r("Small example to encode"      ,return_tensors=TensorType.TENSORFLOW					)      ,_SCREAMING_SNAKE_CASE					)






     @require_torch
     @require_tokenizers
     def        _lowercase			(				self					)							->    Optional[int]:
          import torch

          _snake_case							=   BertTokenizer.from_pretrained("bert-base-cased"					)
          _snake_case							=   BertTokenizerFast.from_pretrained("bert-base-cased"					)

          with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"					):
               self.assert_dump_and_restore(
                   tokenizer_p("Small example to encode"      ,return_tensors=TensorType.PYTORCH					)      ,torch.equal					)

          with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"					):
               self.assert_dump_and_restore(
                   tokenizer_r("Small example to encode"      ,return_tensors=TensorType.PYTORCH					)      ,torch.equal					)






     @require_tokenizers
     def        _lowercase			(				self					)							->    int:
          _snake_case							=   BertTokenizer.from_pretrained("bert-base-cased"					)
          _snake_case							=   BertTokenizerFast.from_pretrained("bert-base-cased"					)

          with self.subTest("Python Tokenizer"					):
               self.assertFalse(tokenizer_p("Small example to_encode"					).is_fast					)

          with self.subTest("Rust Tokenizer"					):
               self.assertTrue(tokenizer_r("Small example to_encode"					).is_fast					)






     @require_tokenizers
     def        _lowercase			(				self					)							->    Dict:
          _snake_case							=   BertTokenizerFast.from_pretrained("bert-base-cased"					)
          _snake_case							=   tokenizer_r(["Test", "\xad", "test"]      ,is_split_into_words=_SCREAMING_SNAKE_CASE					)

          self.assertEqual(encoded.word_to_tokens(0					)      ,TokenSpan(start=1      ,end=2					)					)
          self.assertEqual(encoded.word_to_tokens(1					)      ,_SCREAMING_SNAKE_CASE					)
          self.assertEqual(encoded.word_to_tokens(2					)      ,TokenSpan(start=2      ,end=3					)					)






     def        _lowercase			(				self					)							->    Union[str, Any]:
          _snake_case							=   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
          _snake_case							=   batch.convert_to_tensors(tensor_type="np"					)
          self.assertEqual(tensor_batch["inputs"].shape      ,(2, 3)					)
          self.assertEqual(tensor_batch["labels"].shape      ,(2,)					)
          # test converting the converted
          with CaptureStderr() as cs:
               _snake_case							=   batch.convert_to_tensors(tensor_type="np"					)
          self.assertFalse(len(cs.err					)      ,msg=f"""should have no warning, but got {cs.err}"""					)

          _snake_case							=   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
          _snake_case							=   batch.convert_to_tensors(tensor_type="np"      ,prepend_batch_axis=_SCREAMING_SNAKE_CASE					)
          self.assertEqual(tensor_batch["inputs"].shape      ,(1, 3)					)
          self.assertEqual(tensor_batch["labels"].shape      ,(1,)					)






     @require_torch
     def        _lowercase			(				self					)							->    int:
          _snake_case							=   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
          _snake_case							=   batch.convert_to_tensors(tensor_type="pt"					)
          self.assertEqual(tensor_batch["inputs"].shape      ,(2, 3)					)
          self.assertEqual(tensor_batch["labels"].shape      ,(2,)					)
          # test converting the converted
          with CaptureStderr() as cs:
               _snake_case							=   batch.convert_to_tensors(tensor_type="pt"					)
          self.assertFalse(len(cs.err					)      ,msg=f"""should have no warning, but got {cs.err}"""					)

          _snake_case							=   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
          _snake_case							=   batch.convert_to_tensors(tensor_type="pt"      ,prepend_batch_axis=_SCREAMING_SNAKE_CASE					)
          self.assertEqual(tensor_batch["inputs"].shape      ,(1, 3)					)
          self.assertEqual(tensor_batch["labels"].shape      ,(1,)					)






     @require_tf
     def        _lowercase			(				self					)							->    Union[str, Any]:
          _snake_case							=   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
          _snake_case							=   batch.convert_to_tensors(tensor_type="tf"					)
          self.assertEqual(tensor_batch["inputs"].shape      ,(2, 3)					)
          self.assertEqual(tensor_batch["labels"].shape      ,(2,)					)
          # test converting the converted
          with CaptureStderr() as cs:
               _snake_case							=   batch.convert_to_tensors(tensor_type="tf"					)
          self.assertFalse(len(cs.err					)      ,msg=f"""should have no warning, but got {cs.err}"""					)

          _snake_case							=   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
          _snake_case							=   batch.convert_to_tensors(tensor_type="tf"      ,prepend_batch_axis=_SCREAMING_SNAKE_CASE					)
          self.assertEqual(tensor_batch["inputs"].shape      ,(1, 3)					)
          self.assertEqual(tensor_batch["labels"].shape      ,(1,)					)






     @require_flax
     def        _lowercase			(				self					)							->    List[Any]:
          _snake_case							=   BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
          _snake_case							=   batch.convert_to_tensors(tensor_type="jax"					)
          self.assertEqual(tensor_batch["inputs"].shape      ,(2, 3)					)
          self.assertEqual(tensor_batch["labels"].shape      ,(2,)					)
          # test converting the converted
          with CaptureStderr() as cs:
               _snake_case							=   batch.convert_to_tensors(tensor_type="jax"					)
          self.assertFalse(len(cs.err					)      ,msg=f"""should have no warning, but got {cs.err}"""					)

          _snake_case							=   BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
          _snake_case							=   batch.convert_to_tensors(tensor_type="jax"      ,prepend_batch_axis=_SCREAMING_SNAKE_CASE					)
          self.assertEqual(tensor_batch["inputs"].shape      ,(1, 3)					)
          self.assertEqual(tensor_batch["labels"].shape      ,(1,)					)






     def        _lowercase			(				self					)							->    str:
          _snake_case							=   [{"input_ids": np.array([0, 1, 2]					)}, {"input_ids": np.array([0, 1, 2, 3]					)}]
          _snake_case							=   BertTokenizer.from_pretrained("bert-base-cased"					)

          _snake_case							=   tokenizer.pad(_SCREAMING_SNAKE_CASE      ,padding=_SCREAMING_SNAKE_CASE					)
          self.assertTrue(isinstance(batch["input_ids"]      ,np.ndarray					)					)
          self.assertEqual(batch["input_ids"].tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
          _snake_case							=   tokenizer.pad(_SCREAMING_SNAKE_CASE      ,padding=_SCREAMING_SNAKE_CASE      ,return_tensors="np"					)
          self.assertTrue(isinstance(batch["input_ids"]      ,np.ndarray					)					)
          self.assertEqual(batch["input_ids"].tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)






     @require_torch
     def        _lowercase			(				self					)							->    Tuple:
          import torch

          _snake_case							=   [{"input_ids": torch.tensor([0, 1, 2]					)}, {"input_ids": torch.tensor([0, 1, 2, 3]					)}]
          _snake_case							=   BertTokenizer.from_pretrained("bert-base-cased"					)

          _snake_case							=   tokenizer.pad(_SCREAMING_SNAKE_CASE      ,padding=_SCREAMING_SNAKE_CASE					)
          self.assertTrue(isinstance(batch["input_ids"]      ,torch.Tensor					)					)
          self.assertEqual(batch["input_ids"].tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
          _snake_case							=   tokenizer.pad(_SCREAMING_SNAKE_CASE      ,padding=_SCREAMING_SNAKE_CASE      ,return_tensors="pt"					)
          self.assertTrue(isinstance(batch["input_ids"]      ,torch.Tensor					)					)
          self.assertEqual(batch["input_ids"].tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)






     @require_tf
     def        _lowercase			(				self					)							->    str:
          import tensorflow as tf

          _snake_case							=   [{"input_ids": tf.constant([0, 1, 2]					)}, {"input_ids": tf.constant([0, 1, 2, 3]					)}]
          _snake_case							=   BertTokenizer.from_pretrained("bert-base-cased"					)

          _snake_case							=   tokenizer.pad(_SCREAMING_SNAKE_CASE      ,padding=_SCREAMING_SNAKE_CASE					)
          self.assertTrue(isinstance(batch["input_ids"]      ,tf.Tensor					)					)
          self.assertEqual(batch["input_ids"].numpy().tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
          _snake_case							=   tokenizer.pad(_SCREAMING_SNAKE_CASE      ,padding=_SCREAMING_SNAKE_CASE      ,return_tensors="tf"					)
          self.assertTrue(isinstance(batch["input_ids"]      ,tf.Tensor					)					)
          self.assertEqual(batch["input_ids"].numpy().tolist()      ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)






     @require_tokenizers
     def        _lowercase			(				self					)							->    Optional[Any]:
          _snake_case							=   Tokenizer(WordPiece(unk_token="[UNK]"					)					)
          PreTrainedTokenizerFast(tokenizer_object=_SCREAMING_SNAKE_CASE					)






     @require_tokenizers
     def        _lowercase			(				self					)							->    Dict:
          _snake_case							=   Tokenizer(WordPiece(unk_token="[UNK]"					)					)
          with tempfile.TemporaryDirectory() as tmpdirname:
               bert_tokenizer.save(os.path.join(_SCREAMING_SNAKE_CASE      ,"tokenizer.json"					)					)
               PreTrainedTokenizerFast(tokenizer_file=os.path.join(_SCREAMING_SNAKE_CASE      ,"tokenizer.json"					)					)








'''simple docstring'''





import argparse
import os


UpperCamelCase_		:    Union[str, Any]									=		{
    '''pytorch''': {
        '''1.13''': {
            '''torch''': '''1.13.1''',
            '''torchvision''': '''0.14.1''',
            '''torchaudio''': '''0.13.1''',
            '''python''': 3.9,
            '''cuda''': '''cu116''',
            '''install''': (
                '''python3 -m pip install --no-cache-dir -U torch==1.13.1 torchvision==0.14.1 torchaudio==0.13.1'''
                ''' --extra-index-url https://download.pytorch.org/whl/cu116'''
            ),
            '''base_image''': '''nvidia/cuda:11.6.2-cudnn8-devel-ubuntu20.04''',
        },
        '''1.12''': {
            '''torch''': '''1.12.1''',
            '''torchvision''': '''0.13.1''',
            '''torchaudio''': '''0.12.1''',
            '''python''': 3.9,
            '''cuda''': '''cu113''',
            '''install''': (
                '''python3 -m pip install --no-cache-dir -U torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1'''
                ''' --extra-index-url https://download.pytorch.org/whl/cu113'''
            ),
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''1.11''': {
            '''torch''': '''1.11.0''',
            '''torchvision''': '''0.12.0''',
            '''torchaudio''': '''0.11.0''',
            '''python''': 3.9,
            '''cuda''': '''cu113''',
            '''install''': (
                '''python3 -m pip install --no-cache-dir -U torch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0'''
                ''' --extra-index-url https://download.pytorch.org/whl/cu113'''
            ),
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''1.10''': {
            '''torch''': '''1.10.2''',
            '''torchvision''': '''0.11.3''',
            '''torchaudio''': '''0.10.2''',
            '''python''': 3.9,
            '''cuda''': '''cu113''',
            '''install''': (
                '''python3 -m pip install --no-cache-dir -U torch==1.10.2 torchvision==0.11.3 torchaudio==0.10.2'''
                ''' --extra-index-url https://download.pytorch.org/whl/cu113'''
            ),
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        # torchaudio < 0.10 has no CUDA-enabled binary distributions
        '''1.9''': {
            '''torch''': '''1.9.1''',
            '''torchvision''': '''0.10.1''',
            '''torchaudio''': '''0.9.1''',
            '''python''': 3.9,
            '''cuda''': '''cu111''',
            '''install''': (
                '''python3 -m pip install --no-cache-dir -U torch==1.9.1 torchvision==0.10.1 torchaudio==0.9.1'''
                ''' --extra-index-url https://download.pytorch.org/whl/cu111'''
            ),
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
    },
    '''tensorflow''': {
        '''2.11''': {
            '''tensorflow''': '''2.11.1''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.11.1''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.10''': {
            '''tensorflow''': '''2.10.1''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.10.1''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.9''': {
            '''tensorflow''': '''2.9.3''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.9.3''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.8''': {
            '''tensorflow''': '''2.8.2''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.8.2''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.7''': {
            '''tensorflow''': '''2.7.3''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.7.3''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.6''': {
            '''tensorflow''': '''2.6.5''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.6.5''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.5''': {
            '''tensorflow''': '''2.5.3''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.5.3''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
    },
}


if __name__ == "__main__":
   UpperCamelCase_		:    Optional[Any]									=		argparse.ArgumentParser('''Choose the framework and version to install''')
   parser.add_argument(
       '''--framework''', help='''The framework to install. Should be `torch` or `tensorflow`''', type=str, required=True
   )
   parser.add_argument('''--version''', help='''The version of the framework to install.''', type=str, required=True)
   UpperCamelCase_		:    Tuple									=		parser.parse_args()

   UpperCamelCase_		:    Optional[int]									=		past_versions_testing[args.framework][args.version]

   os.system(F'echo "export INSTALL_CMD=\'{info["install"]}\'" >> ~/.profile')
   print(F'echo "export INSTALL_CMD=\'{info["install"]}\'" >> ~/.profile')

   UpperCamelCase_		:    List[Any]									=		''''''
   if args.framework == "pytorch":
      UpperCamelCase_		:    str									=		info['''cuda''']
   os.system(F'echo \"export CUDA=\'{cuda}\'\" >> ~/.profile')
   print(F'echo \"export CUDA=\'{cuda}\'\" >> ~/.profile')


