






"""simple docstring"""





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
   from tokenizers import Tokenizer
   from tokenizers.models import WordPiece




class       _lowerCAmelCase						(			unittest.TestCase     ):







 """simple docstring"""







 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    Optional[int]			):
    __lowercase			=		list(tokenizer_class.max_model_input_sizes.keys()			)
    for model_name in sa_models[:1]:
       __lowercase			=		tokenizer_class.from_pretrained(UpperCAmelCase__			)
       self.assertIsNotNone(UpperCAmelCase__			)
       self.assertIsInstance(UpperCAmelCase__,      UpperCAmelCase__			)
       self.assertIsInstance(UpperCAmelCase__,      UpperCAmelCase__			)

       for special_tok in tokenizer.all_special_tokens:
          self.assertIsInstance(UpperCAmelCase__,      UpperCAmelCase__			)
          __lowercase			=		tokenizer.convert_tokens_to_ids(UpperCAmelCase__			)
          self.assertIsInstance(UpperCAmelCase__,      UpperCAmelCase__			)


 def       _lowercase				(    self							:    Union[str, Any],      UpperCAmelCase__							:    BatchEncoding,      UpperCAmelCase__							:    Optional[Callable] = None			):
    __lowercase			=		pickle.dumps(UpperCAmelCase__			)
    self.assertIsNotNone(UpperCAmelCase__			)

    __lowercase			=		pickle.loads(UpperCAmelCase__			)

    # Ensure is_fast is correctly restored
    self.assertEqual(be_restored.is_fast,      be_original.is_fast			)

    # Ensure encodings are potentially correctly restored
    if be_original.is_fast:
       self.assertIsNotNone(be_restored.encodings			)
    else:
       self.assertIsNone(be_restored.encodings			)

    # Ensure the keys are the same
    for original_v, restored_v in zip(be_original.values(),      be_restored.values()			):
       if equal_op:
          self.assertTrue(equal_op(UpperCAmelCase__,      UpperCAmelCase__			)			)
       else:
          self.assertEqual(UpperCAmelCase__,      UpperCAmelCase__			)


 @slow
 def       _lowercase				(    self							:    List[Any]			):
    self.check_tokenizer_from_pretrained(UpperCAmelCase__			)


 def       _lowercase				(    self							:    str			):
    self.assertEqual(TensorType("tf"			),      TensorType.TENSORFLOW			)
    self.assertEqual(TensorType("pt"			),      TensorType.PYTORCH			)
    self.assertEqual(TensorType("np"			),      TensorType.NUMPY			)


 @require_tokenizers
 def       _lowercase				(    self							:    str			):
    import numpy as np

    __lowercase			=		BertTokenizer.from_pretrained("bert-base-cased"			)
    __lowercase			=		BertTokenizerFast.from_pretrained("bert-base-cased"			)

    # Python no tensor
    with self.subTest("BatchEncoding (Python, return_tensors=None)"			):
       self.assert_dump_and_restore(tokenizer_p("Small example to encode"			)			)

    with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"			):
       self.assert_dump_and_restore(
           tokenizer_p("Small example to encode",      return_tensors=TensorType.NUMPY			),      np.array_equal			)

    with self.subTest("BatchEncoding (Rust, return_tensors=None)"			):
       self.assert_dump_and_restore(tokenizer_r("Small example to encode"			)			)

    with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"			):
       self.assert_dump_and_restore(
           tokenizer_r("Small example to encode",      return_tensors=TensorType.NUMPY			),      np.array_equal			)


 @require_tf
 @require_tokenizers
 def       _lowercase				(    self							:    List[str]			):
    import tensorflow as tf

    def tf_array_equals(UpperCAmelCase__							:    Any,      UpperCAmelCase__							:    Union[str, Any]			):
       return tf.reduce_all(tf.equal(UpperCAmelCase__,      UpperCAmelCase__			)			)

    __lowercase			=		BertTokenizer.from_pretrained("bert-base-cased"			)
    __lowercase			=		BertTokenizerFast.from_pretrained("bert-base-cased"			)

    with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"			):
       self.assert_dump_and_restore(
           tokenizer_p("Small example to encode",      return_tensors=TensorType.TENSORFLOW			),      UpperCAmelCase__			)

    with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"			):
       self.assert_dump_and_restore(
           tokenizer_r("Small example to encode",      return_tensors=TensorType.TENSORFLOW			),      UpperCAmelCase__			)


 @require_torch
 @require_tokenizers
 def       _lowercase				(    self							:    str			):
    import torch

    __lowercase			=		BertTokenizer.from_pretrained("bert-base-cased"			)
    __lowercase			=		BertTokenizerFast.from_pretrained("bert-base-cased"			)

    with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"			):
       self.assert_dump_and_restore(
           tokenizer_p("Small example to encode",      return_tensors=TensorType.PYTORCH			),      torch.equal			)

    with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"			):
       self.assert_dump_and_restore(
           tokenizer_r("Small example to encode",      return_tensors=TensorType.PYTORCH			),      torch.equal			)


 @require_tokenizers
 def       _lowercase				(    self							:    Union[str, Any]			):
    __lowercase			=		BertTokenizer.from_pretrained("bert-base-cased"			)
    __lowercase			=		BertTokenizerFast.from_pretrained("bert-base-cased"			)

    with self.subTest("Python Tokenizer"			):
       self.assertFalse(tokenizer_p("Small example to_encode"			).is_fast			)

    with self.subTest("Rust Tokenizer"			):
       self.assertTrue(tokenizer_r("Small example to_encode"			).is_fast			)


 @require_tokenizers
 def       _lowercase				(    self							:    Dict			):
    __lowercase			=		BertTokenizerFast.from_pretrained("bert-base-cased"			)
    __lowercase			=		tokenizer_r(["Test", "\xad", "test"],      is_split_into_words=UpperCAmelCase__			)

    self.assertEqual(encoded.word_to_tokens(0			),      TokenSpan(start=1,      end=2			)			)
    self.assertEqual(encoded.word_to_tokens(1			),      UpperCAmelCase__			)
    self.assertEqual(encoded.word_to_tokens(2			),      TokenSpan(start=2,      end=3			)			)


 def       _lowercase				(    self							:    Dict			):
    __lowercase			=		BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}			)
    __lowercase			=		batch.convert_to_tensors(tensor_type="np"			)
    self.assertEqual(tensor_batch["inputs"].shape,      (2, 3)			)
    self.assertEqual(tensor_batch["labels"].shape,      (2,)			)
    # test converting the converted
    with CaptureStderr() as cs:
       __lowercase			=		batch.convert_to_tensors(tensor_type="np"			)
    self.assertFalse(len(cs.err			),      msg=F"""should have no warning, but got {cs.err}"""			)

    __lowercase			=		BatchEncoding({"inputs": [1, 2, 3], "labels": 0}			)
    __lowercase			=		batch.convert_to_tensors(tensor_type="np",      prepend_batch_axis=UpperCAmelCase__			)
    self.assertEqual(tensor_batch["inputs"].shape,      (1, 3)			)
    self.assertEqual(tensor_batch["labels"].shape,      (1,)			)


 @require_torch
 def       _lowercase				(    self							:    List[str]			):
    __lowercase			=		BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}			)
    __lowercase			=		batch.convert_to_tensors(tensor_type="pt"			)
    self.assertEqual(tensor_batch["inputs"].shape,      (2, 3)			)
    self.assertEqual(tensor_batch["labels"].shape,      (2,)			)
    # test converting the converted
    with CaptureStderr() as cs:
       __lowercase			=		batch.convert_to_tensors(tensor_type="pt"			)
    self.assertFalse(len(cs.err			),      msg=F"""should have no warning, but got {cs.err}"""			)

    __lowercase			=		BatchEncoding({"inputs": [1, 2, 3], "labels": 0}			)
    __lowercase			=		batch.convert_to_tensors(tensor_type="pt",      prepend_batch_axis=UpperCAmelCase__			)
    self.assertEqual(tensor_batch["inputs"].shape,      (1, 3)			)
    self.assertEqual(tensor_batch["labels"].shape,      (1,)			)


 @require_tf
 def       _lowercase				(    self							:    str			):
    __lowercase			=		BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}			)
    __lowercase			=		batch.convert_to_tensors(tensor_type="tf"			)
    self.assertEqual(tensor_batch["inputs"].shape,      (2, 3)			)
    self.assertEqual(tensor_batch["labels"].shape,      (2,)			)
    # test converting the converted
    with CaptureStderr() as cs:
       __lowercase			=		batch.convert_to_tensors(tensor_type="tf"			)
    self.assertFalse(len(cs.err			),      msg=F"""should have no warning, but got {cs.err}"""			)

    __lowercase			=		BatchEncoding({"inputs": [1, 2, 3], "labels": 0}			)
    __lowercase			=		batch.convert_to_tensors(tensor_type="tf",      prepend_batch_axis=UpperCAmelCase__			)
    self.assertEqual(tensor_batch["inputs"].shape,      (1, 3)			)
    self.assertEqual(tensor_batch["labels"].shape,      (1,)			)


 @require_flax
 def       _lowercase				(    self							:    List[str]			):
    __lowercase			=		BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}			)
    __lowercase			=		batch.convert_to_tensors(tensor_type="jax"			)
    self.assertEqual(tensor_batch["inputs"].shape,      (2, 3)			)
    self.assertEqual(tensor_batch["labels"].shape,      (2,)			)
    # test converting the converted
    with CaptureStderr() as cs:
       __lowercase			=		batch.convert_to_tensors(tensor_type="jax"			)
    self.assertFalse(len(cs.err			),      msg=F"""should have no warning, but got {cs.err}"""			)

    __lowercase			=		BatchEncoding({"inputs": [1, 2, 3], "labels": 0}			)
    __lowercase			=		batch.convert_to_tensors(tensor_type="jax",      prepend_batch_axis=UpperCAmelCase__			)
    self.assertEqual(tensor_batch["inputs"].shape,      (1, 3)			)
    self.assertEqual(tensor_batch["labels"].shape,      (1,)			)


 def       _lowercase				(    self							:    List[str]			):
    __lowercase			=		[{"input_ids": np.array([0, 1, 2]			)}, {"input_ids": np.array([0, 1, 2, 3]			)}]
    __lowercase			=		BertTokenizer.from_pretrained("bert-base-cased"			)

    __lowercase			=		tokenizer.pad(UpperCAmelCase__,      padding=UpperCAmelCase__			)
    self.assertTrue(isinstance(batch["input_ids"],      np.ndarray			)			)
    self.assertEqual(batch["input_ids"].tolist(),      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)
    __lowercase			=		tokenizer.pad(UpperCAmelCase__,      padding=UpperCAmelCase__,      return_tensors="np"			)
    self.assertTrue(isinstance(batch["input_ids"],      np.ndarray			)			)
    self.assertEqual(batch["input_ids"].tolist(),      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)


 @require_torch
 def       _lowercase				(    self							:    str			):
    import torch

    __lowercase			=		[{"input_ids": torch.tensor([0, 1, 2]			)}, {"input_ids": torch.tensor([0, 1, 2, 3]			)}]
    __lowercase			=		BertTokenizer.from_pretrained("bert-base-cased"			)

    __lowercase			=		tokenizer.pad(UpperCAmelCase__,      padding=UpperCAmelCase__			)
    self.assertTrue(isinstance(batch["input_ids"],      torch.Tensor			)			)
    self.assertEqual(batch["input_ids"].tolist(),      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)
    __lowercase			=		tokenizer.pad(UpperCAmelCase__,      padding=UpperCAmelCase__,      return_tensors="pt"			)
    self.assertTrue(isinstance(batch["input_ids"],      torch.Tensor			)			)
    self.assertEqual(batch["input_ids"].tolist(),      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)


 @require_tf
 def       _lowercase				(    self							:    List[str]			):
    import tensorflow as tf

    __lowercase			=		[{"input_ids": tf.constant([0, 1, 2]			)}, {"input_ids": tf.constant([0, 1, 2, 3]			)}]
    __lowercase			=		BertTokenizer.from_pretrained("bert-base-cased"			)

    __lowercase			=		tokenizer.pad(UpperCAmelCase__,      padding=UpperCAmelCase__			)
    self.assertTrue(isinstance(batch["input_ids"],      tf.Tensor			)			)
    self.assertEqual(batch["input_ids"].numpy().tolist(),      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)
    __lowercase			=		tokenizer.pad(UpperCAmelCase__,      padding=UpperCAmelCase__,      return_tensors="tf"			)
    self.assertTrue(isinstance(batch["input_ids"],      tf.Tensor			)			)
    self.assertEqual(batch["input_ids"].numpy().tolist(),      [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]			)


 @require_tokenizers
 def       _lowercase				(    self							:    Optional[int]			):
    __lowercase			=		Tokenizer(WordPiece(unk_token="[UNK]"			)			)
    PreTrainedTokenizerFast(tokenizer_object=UpperCAmelCase__			)



 @require_tokenizers
 def       _lowercase				(    self							:    Optional[int]			):
    __lowercase			=		Tokenizer(WordPiece(unk_token="[UNK]"			)			)
    with tempfile.TemporaryDirectory() as tmpdirname:
       bert_tokenizer.save(os.path.join(UpperCAmelCase__,      "tokenizer.json"			)			)
       PreTrainedTokenizerFast(tokenizer_file=os.path.join(UpperCAmelCase__,      "tokenizer.json"			)			)







"""simple docstring"""





import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		Tuple:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file.jsonl"
   __lowercase			=		textwrap.dedent(
       "\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		Any:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_utf16_encoded.jsonl"
   __lowercase			=		textwrap.dedent(
       "\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        ")
   with open(UpperCamelCase_,    "w",    encoding="utf-16") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : Optional[int])			->		Optional[Any]:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_with_list_of_dicts.json"
   __lowercase			=		textwrap.dedent(
       "\\n        [\n            {\"col_1\": -1},\n            {\"col_1\": 1, \"col_2\": 2},\n            {\"col_1\": 10, \"col_2\": 20}\n        ]\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		int:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_with_list_of_dicts_field.json"
   __lowercase			=		textwrap.dedent(
       "\\n        {\n            \"field1\": 1,\n            \"field2\": \"aabb\",\n            \"field3\": [\n                {\"col_1\": -1},\n                {\"col_1\": 1, \"col_2\": 2},\n                {\"col_1\": 10, \"col_2\": 20}\n            ]\n        }\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.mark.parametrize(
    "file_fixture, config_kwargs",    [
        ("jsonl_file", {}),
        ("jsonl_file_utf16_encoded", {"encoding": "utf-16"}),
        ("json_file_with_list_of_dicts", {}),
        ("json_file_with_list_of_dicts_field", {"field": "field3"}),
    ],    )
def 						_A						(						UpperCamelCase_  : Any,    UpperCamelCase_  : List[Any],    UpperCamelCase_  : Optional[Any])			->		List[Any]:
   '''simple docstring'''


   __lowercase			=		Json(**UpperCamelCase_)
   __lowercase			=		json._generate_tables([[request.getfixturevalue(UpperCamelCase_)]])
   __lowercase			=		pa.concat_tables([table for _, table in generator])
   assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}




@pytest.mark.parametrize(
    "file_fixture, config_kwargs",    [
        (
            "jsonl_file",
            {"features": Features({"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")})},
        ),
        (
            "json_file_with_list_of_dicts",
            {"features": Features({"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")})},
        ),
        (
            "json_file_with_list_of_dicts_field",
            {
                "field": "field3",
                "features": Features(
                    {"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")}),
            },
        ),
    ],    )
def 						_A						(						UpperCamelCase_  : Optional[int],    UpperCamelCase_  : int,    UpperCamelCase_  : str)			->		Optional[int]:
   '''simple docstring'''


   __lowercase			=		Json(**UpperCamelCase_)
   __lowercase			=		json._generate_tables([[request.getfixturevalue(UpperCamelCase_)]])
   __lowercase			=		pa.concat_tables([table for _, table in generator])
   assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}




