

"""simple docstring"""





import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


_a				   =							logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
_a				   =							direct_transformers_import(Path(__file__).parent)


_a				   =							{
    'AutoTokenizer': 'PreTrainedTokenizerBase',
    'AutoFeatureExtractor': 'FeatureExtractionMixin',
    'AutoImageProcessor': 'ImageProcessingMixin',
}




class       _lowerCAmelCase						(			lowercase     ):







 """simple docstring"""







 __UpperCAmelCase			:      Optional[int]     	=			["feature_extractor", "tokenizer"]
 # Names need to be attr_class for attr in attributes
 __UpperCAmelCase			:      Union[str, Any]     	=			None
 __UpperCAmelCase			:      Optional[int]     	=			None
 __UpperCAmelCase			:      Any     	=			None


 def __init__(    self							:    Union[str, Any],      *UpperCAmelCase__							:    Any,      **UpperCAmelCase__							:    str			):
    # Sanitize args and kwargs
    for key in kwargs:
       if key not in self.attributes:
          raise TypeError(F"""Unexpected keyword argument {key}."""			)
    for arg, attribute_name in zip(UpperCAmelCase__,      self.attributes			):
       if attribute_name in kwargs:
          raise TypeError(F"""Got multiple values for argument {attribute_name}."""			)
       else:
          __lowercase			=		arg

    if len(UpperCAmelCase__			) != len(self.attributes			):
       raise ValueError(
           F"""This processor requires {len(self.attributes			)} arguments: {", ".join(self.attributes			)}. Got """
           F"""{len(UpperCAmelCase__			)} arguments instead."""			)

    # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
    for attribute_name, arg in kwargs.items():
       __lowercase			=		getattr(self,      F"""{attribute_name}_class"""			)
       # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
       __lowercase			=		AUTO_TO_BASE_CLASS_MAPPING.get(UpperCAmelCase__,      UpperCAmelCase__			)
       if isinstance(UpperCAmelCase__,      UpperCAmelCase__			):
          __lowercase			=		tuple(getattr(UpperCAmelCase__,      UpperCAmelCase__			) for n in class_name if n is not None			)
       else:
          __lowercase			=		getattr(UpperCAmelCase__,      UpperCAmelCase__			)

       if not isinstance(UpperCAmelCase__,      UpperCAmelCase__			):
          raise ValueError(
              F"""Received a {type(UpperCAmelCase__			).__name__} for argument {attribute_name}, but a {class_name} was expected."""			)

       setattr(self,      UpperCAmelCase__,      UpperCAmelCase__			)


 def __repr__(    self							:    Optional[int]			):
    __lowercase			=		[F"""- {name}: {repr(getattr(self,      UpperCAmelCase__			)			)}""" for name in self.attributes]
    __lowercase			=		"\n".join(UpperCAmelCase__			)
    return F"""{self.__class__.__name__}:\n{attributes_repr}"""


 def       _lowercase				(    self							:    str,      UpperCAmelCase__							:    Optional[int],      UpperCAmelCase__							:    bool = False,      **UpperCAmelCase__							:    Optional[Any]			):
    os.makedirs(UpperCAmelCase__,      exist_ok=UpperCAmelCase__			)

    if push_to_hub:
       __lowercase			=		kwargs.pop("commit_message",      UpperCAmelCase__			)
       __lowercase			=		kwargs.pop("repo_id",      save_directory.split(os.path.sep			)[-1]			)
       __lowercase			=		self._create_repo(UpperCAmelCase__,      **UpperCAmelCase__			)
       __lowercase			=		self._get_files_timestamps(UpperCAmelCase__			)
    # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
    # loaded from the Hub.
    if self._auto_class is not None:
       __lowercase			=		[getattr(self,      UpperCAmelCase__			) for attribute_name in self.attributes]
       __lowercase			=		[(a.init_kwargs if isinstance(UpperCAmelCase__,      UpperCAmelCase__			) else a) for a in attrs]
       custom_object_save(self,      UpperCAmelCase__,      config=UpperCAmelCase__			)

    for attribute_name in self.attributes:
       __lowercase			=		getattr(self,      UpperCAmelCase__			)
       # Include the processor class in the attribute config so this processor can then be reloaded with the
       # `AutoProcessor` API.
       if hasattr(UpperCAmelCase__,      "_set_processor_class"			):
          attribute._set_processor_class(self.__class__.__name__			)
       attribute.save_pretrained(UpperCAmelCase__			)

    if self._auto_class is not None:
       # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
       for attribute_name in self.attributes:
          __lowercase			=		getattr(self,      UpperCAmelCase__			)
          if isinstance(UpperCAmelCase__,      UpperCAmelCase__			):
             del attribute.init_kwargs["auto_map"]

    if push_to_hub:
       self._upload_modified_files(
           UpperCAmelCase__,      UpperCAmelCase__,      UpperCAmelCase__,      commit_message=UpperCAmelCase__,      token=kwargs.get("use_auth_token"			),      )


 @classmethod
 def       _lowercase				(    cls							:    Any,      UpperCAmelCase__							:    Union[str, os.PathLike],      UpperCAmelCase__							:    Optional[Union[str, os.PathLike]] = None,      UpperCAmelCase__							:    bool = False,      UpperCAmelCase__							:    bool = False,      UpperCAmelCase__							:    Optional[Union[str, bool]] = None,      UpperCAmelCase__							:    str = "main",      **UpperCAmelCase__							:    Dict,      ):
    __lowercase			=		cache_dir
    __lowercase			=		force_download
    __lowercase			=		local_files_only
    __lowercase			=		revision

    __lowercase			=		kwargs.pop("use_auth_token",      UpperCAmelCase__			)
    if use_auth_token is not None:
       warnings.warn(
           "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.",      UpperCAmelCase__			)
       if token is not None:
          raise ValueError(
              "`token` and `use_auth_token` are both specified. Please set only the argument `token`."			)
       __lowercase			=		use_auth_token

    if token is not None:
       # change to `token` in a follow-up PR
       __lowercase			=		token

    __lowercase			=		cls._get_arguments_from_pretrained(UpperCAmelCase__,      **UpperCAmelCase__			)
    return cls(*UpperCAmelCase__			)


 @classmethod
 def       _lowercase				(    cls							:    int,      UpperCAmelCase__							:    Any="AutoProcessor"			):
    if not isinstance(UpperCAmelCase__,      UpperCAmelCase__			):
       __lowercase			=		auto_class.__name__

    import transformers.models.auto as auto_module

    if not hasattr(UpperCAmelCase__,      UpperCAmelCase__			):
       raise ValueError(F"""{auto_class} is not a valid auto class."""			)

    __lowercase			=		auto_class


 @classmethod
 def       _lowercase				(    cls							:    Dict,      UpperCAmelCase__							:    Dict,      **UpperCAmelCase__							:    Any			):
    __lowercase			=		[]
    for attribute_name in cls.attributes:
       __lowercase			=		getattr(cls,      F"""{attribute_name}_class"""			)
       if isinstance(UpperCAmelCase__,      UpperCAmelCase__			):
          __lowercase			=		tuple(getattr(UpperCAmelCase__,      UpperCAmelCase__			) if n is not None else None for n in class_name			)
          __lowercase			=		kwargs.get("use_fast",      UpperCAmelCase__			)
          if use_fast and classes[1] is not None:
             __lowercase			=		classes[1]
          else:
             __lowercase			=		classes[0]
       else:
          __lowercase			=		getattr(UpperCAmelCase__,      UpperCAmelCase__			)

       args.append(attribute_class.from_pretrained(UpperCAmelCase__,      **UpperCAmelCase__			)			)
    return args



 @property
 def       _lowercase				(    self							:    List[str]			):
    __lowercase			=		getattr(self,      self.attributes[0]			)
    return getattr(UpperCAmelCase__,      "model_input_names",      UpperCAmelCase__			)







_a				   =							copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
   _a				   =							ProcessorMixin.push_to_hub.__doc__.format(
       object='processor', object_class='AutoProcessor', object_files='processor files'
   )







"""simple docstring"""





import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		Tuple:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file.jsonl"
   __lowercase			=		textwrap.dedent(
       "\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		Any:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_utf16_encoded.jsonl"
   __lowercase			=		textwrap.dedent(
       "\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        ")
   with open(UpperCamelCase_,    "w",    encoding="utf-16") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : Optional[int])			->		Optional[Any]:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_with_list_of_dicts.json"
   __lowercase			=		textwrap.dedent(
       "\\n        [\n            {\"col_1\": -1},\n            {\"col_1\": 1, \"col_2\": 2},\n            {\"col_1\": 10, \"col_2\": 20}\n        ]\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		int:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_with_list_of_dicts_field.json"
   __lowercase			=		textwrap.dedent(
       "\\n        {\n            \"field1\": 1,\n            \"field2\": \"aabb\",\n            \"field3\": [\n                {\"col_1\": -1},\n                {\"col_1\": 1, \"col_2\": 2},\n                {\"col_1\": 10, \"col_2\": 20}\n            ]\n        }\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.mark.parametrize(
    "file_fixture, config_kwargs",    [
        ("jsonl_file", {}),
        ("jsonl_file_utf16_encoded", {"encoding": "utf-16"}),
        ("json_file_with_list_of_dicts", {}),
        ("json_file_with_list_of_dicts_field", {"field": "field3"}),
    ],    )
def 						_A						(						UpperCamelCase_  : Any,    UpperCamelCase_  : List[Any],    UpperCamelCase_  : Optional[Any])			->		List[Any]:
   '''simple docstring'''


   __lowercase			=		Json(**UpperCamelCase_)
   __lowercase			=		json._generate_tables([[request.getfixturevalue(UpperCamelCase_)]])
   __lowercase			=		pa.concat_tables([table for _, table in generator])
   assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}




@pytest.mark.parametrize(
    "file_fixture, config_kwargs",    [
        (
            "jsonl_file",
            {"features": Features({"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")})},
        ),
        (
            "json_file_with_list_of_dicts",
            {"features": Features({"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")})},
        ),
        (
            "json_file_with_list_of_dicts_field",
            {
                "field": "field3",
                "features": Features(
                    {"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")}),
            },
        ),
    ],    )
def 						_A						(						UpperCamelCase_  : Optional[int],    UpperCamelCase_  : int,    UpperCamelCase_  : str)			->		Optional[int]:
   '''simple docstring'''


   __lowercase			=		Json(**UpperCamelCase_)
   __lowercase			=		json._generate_tables([[request.getfixturevalue(UpperCamelCase_)]])
   __lowercase			=		pa.concat_tables([table for _, table in generator])
   assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}




