

"""simple docstring"""





import glob
import os
from typing import Dict, List, Union

import torch

from diffusers.utils import is_safetensors_available


if is_safetensors_available():
   import safetensors.torch

from huggingface_hub import snapshot_download

from diffusers import DiffusionPipeline, __version__
from diffusers.schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME
from diffusers.utils import CONFIG_NAME, DIFFUSERS_CACHE, ONNX_WEIGHTS_NAME, WEIGHTS_NAME




class       _lowerCAmelCase						(			lowercase     ):







 """simple docstring"""







 def __init__(    self							:    Optional[Any]			):
    self.register_to_config()
    super().__init__()


 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    Any,      UpperCAmelCase__							:    List[Any]			):
    if dicta == dicta:
       return True
    else:
       __lowercase	,__lowercase			=		self._remove_meta_keys(UpperCAmelCase__			)
       __lowercase	,__lowercase			=		self._remove_meta_keys(UpperCAmelCase__			)
       if configa == configa:
          print(F"""Warning !: Mismatch in keys {meta_keysa} and {meta_keysa}."""			)
          return True
    return False


 def       _lowercase				(    self							:    Union[str, Any],      UpperCAmelCase__							:    Dict			):
    __lowercase			=		[]
    __lowercase			=		config_dict.copy()
    for key in config_dict.keys():
       if key.startswith("_"			):
          temp_dict.pop(UpperCAmelCase__			)
          meta_keys.append(UpperCAmelCase__			)
    return (temp_dict, meta_keys)


 @torch.no_grad()
 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    List[Union[str, os.PathLike]],      **UpperCAmelCase__							:    Union[str, Any]			):
    __lowercase			=		kwargs.pop("cache_dir",      UpperCAmelCase__			)
    __lowercase			=		kwargs.pop("resume_download",      UpperCAmelCase__			)
    __lowercase			=		kwargs.pop("force_download",      UpperCAmelCase__			)
    __lowercase			=		kwargs.pop("proxies",      UpperCAmelCase__			)
    __lowercase			=		kwargs.pop("local_files_only",      UpperCAmelCase__			)
    __lowercase			=		kwargs.pop("use_auth_token",      UpperCAmelCase__			)
    __lowercase			=		kwargs.pop("revision",      UpperCAmelCase__			)
    __lowercase			=		kwargs.pop("torch_dtype",      UpperCAmelCase__			)
    __lowercase			=		kwargs.pop("device_map",      UpperCAmelCase__			)

    __lowercase			=		kwargs.pop("alpha",      0.5			)
    __lowercase			=		kwargs.pop("interp",      UpperCAmelCase__			)

    print("Received list",      UpperCAmelCase__			)
    print(F"""Combining with alpha={alpha}, interpolation mode={interp}"""			)

    __lowercase			=		len(UpperCAmelCase__			)
    # Ignore result from model_index_json comparision of the two checkpoints
    __lowercase			=		kwargs.pop("force",      UpperCAmelCase__			)

    # If less than 2 checkpoints, nothing to merge. If more than 3, not supported for now.
    if checkpoint_count > 3 or checkpoint_count < 2:
       raise ValueError(
           "Received incorrect number of checkpoints to merge. Ensure that either 2 or 3 checkpoints are being"
           " passed."			)

    print("Received the right number of checkpoints"			)
    # chkpt0, chkpt1 = pretrained_model_name_or_path_list[0:2]
    # chkpt2 = pretrained_model_name_or_path_list[2] if checkpoint_count == 3 else None

    # Validate that the checkpoints can be merged
    # Step 1: Load the model config and compare the checkpoints. We'll compare the model_index.json first while ignoring the keys starting with '_'
    __lowercase			=		[]
    for pretrained_model_name_or_path in pretrained_model_name_or_path_list:
       __lowercase			=		DiffusionPipeline.load_config(
           UpperCAmelCase__,      cache_dir=UpperCAmelCase__,      resume_download=UpperCAmelCase__,      force_download=UpperCAmelCase__,      proxies=UpperCAmelCase__,      local_files_only=UpperCAmelCase__,      use_auth_token=UpperCAmelCase__,      revision=UpperCAmelCase__,      )
       config_dicts.append(UpperCAmelCase__			)

    __lowercase			=		True
    for idx in range(1,      len(UpperCAmelCase__			)			):
       comparison_result &= self._compare_model_configs(config_dicts[idx - 1],      config_dicts[idx]			)
       if not force and comparison_result is False:
          raise ValueError("Incompatible checkpoints. Please check model_index.json for the models."			)
          print(config_dicts[0],      config_dicts[1]			)
    print("Compatible model_index.json files found"			)
    # Step 2: Basic Validation has succeeded. Let's download the models and save them into our local files.
    __lowercase			=		[]
    for pretrained_model_name_or_path, config_dict in zip(UpperCAmelCase__,      UpperCAmelCase__			):
       __lowercase			=		[k for k in config_dict.keys() if not k.startswith("_"			)]
       __lowercase			=		[os.path.join(UpperCAmelCase__,      "*"			) for k in folder_names]
       allow_patterns += [
           WEIGHTS_NAME,
           SCHEDULER_CONFIG_NAME,
           CONFIG_NAME,
           ONNX_WEIGHTS_NAME,
           DiffusionPipeline.config_name,
       ]
       __lowercase			=		config_dict.get("_class_name"			)
       __lowercase			=		{"diffusers": __version__, "pipeline_class": requested_pipeline_class}

       __lowercase			=		(
           pretrained_model_name_or_path
           if os.path.isdir(UpperCAmelCase__			)
           else snapshot_download(
               UpperCAmelCase__,      cache_dir=UpperCAmelCase__,      resume_download=UpperCAmelCase__,      proxies=UpperCAmelCase__,      local_files_only=UpperCAmelCase__,      use_auth_token=UpperCAmelCase__,      revision=UpperCAmelCase__,      allow_patterns=UpperCAmelCase__,      user_agent=UpperCAmelCase__,      )
       )
       print("Cached Folder",      UpperCAmelCase__			)
       cached_folders.append(UpperCAmelCase__			)

    # Step 3:-
    # Load the first checkpoint as a diffusion pipeline and modify its module state_dict in place
    __lowercase			=		DiffusionPipeline.from_pretrained(
        cached_folders[0],      torch_dtype=UpperCAmelCase__,      device_map=UpperCAmelCase__			)
    final_pipe.to(self.device			)

    __lowercase			=		None
    if len(UpperCAmelCase__			) > 2:
       __lowercase			=		os.path.join(cached_folders[2]			)

    if interp == "sigmoid":
       __lowercase			=		CheckpointMergerPipeline.sigmoid
    elif interp == "inv_sigmoid":
       __lowercase			=		CheckpointMergerPipeline.inv_sigmoid
    elif interp == "add_diff":
       __lowercase			=		CheckpointMergerPipeline.add_difference
    else:
       __lowercase			=		CheckpointMergerPipeline.weighted_sum

    # Find each module's state dict.
    for attr in final_pipe.config.keys():
       if not attr.startswith("_"			):
          __lowercase			=		os.path.join(cached_folders[1],      UpperCAmelCase__			)
          if os.path.exists(UpperCAmelCase__			):
             __lowercase			=		[
                 *glob.glob(os.path.join(UpperCAmelCase__,      "*.safetensors"			)			),
                 *glob.glob(os.path.join(UpperCAmelCase__,      "*.bin"			)			),
             ]
             __lowercase			=		files[0] if len(UpperCAmelCase__			) > 0 else None
          if len(UpperCAmelCase__			) < 3:
             __lowercase			=		None
          else:
             __lowercase			=		os.path.join(cached_folders[2],      UpperCAmelCase__			)
             if os.path.exists(UpperCAmelCase__			):
                __lowercase			=		[
                    *glob.glob(os.path.join(UpperCAmelCase__,      "*.safetensors"			)			),
                    *glob.glob(os.path.join(UpperCAmelCase__,      "*.bin"			)			),
                ]
                __lowercase			=		files[0] if len(UpperCAmelCase__			) > 0 else None
                # For an attr if both checkpoint_path_1 and 2 are None, ignore.
                # If atleast one is present, deal with it according to interp method, of course only if the state_dict keys match.
          if checkpoint_path_a is None and checkpoint_path_a is None:
             print(F"""Skipping {attr}: not present in 2nd or 3d model"""			)
             continue
          try:
             __lowercase			=		getattr(UpperCAmelCase__,      UpperCAmelCase__			)
             if isinstance(UpperCAmelCase__,      UpperCAmelCase__			):  # ignore requires_safety_checker boolean
                continue
             __lowercase			=		getattr(UpperCAmelCase__,      "state_dict"			)
             __lowercase			=		theta_a()

             __lowercase			=		getattr(UpperCAmelCase__,      "load_state_dict"			)
             __lowercase			=		(
                 safetensors.torch.load_file(UpperCAmelCase__			)
                 if (is_safetensors_available() and checkpoint_path_a.endswith(".safetensors"			))
                 else torch.load(UpperCAmelCase__,      map_location="cpu"			)
             )
             __lowercase			=		None
             if checkpoint_path_a:
                __lowercase			=		(
                    safetensors.torch.load_file(UpperCAmelCase__			)
                    if (is_safetensors_available() and checkpoint_path_a.endswith(".safetensors"			))
                    else torch.load(UpperCAmelCase__,      map_location="cpu"			)
                )

             if not theta_a.keys() == theta_a.keys():
                print(F"""Skipping {attr}: key mismatch"""			)
                continue
             if theta_a and not theta_a.keys() == theta_a.keys():
                print(F"""Skipping {attr}:y mismatch"""			)
          except Exception as e:
             print(F"""Skipping {attr} do to an unexpected error: {str(UpperCAmelCase__			)}"""			)
             continue
          print(F"""MERGING {attr}"""			)

          for key in theta_a.keys():
             if theta_a:
                __lowercase			=		theta_func(theta_a[key],      theta_a[key],      theta_a[key],      UpperCAmelCase__			)
             else:
                __lowercase			=		theta_func(theta_a[key],      theta_a[key],      UpperCAmelCase__,      UpperCAmelCase__			)

          del theta_a
          del theta_a
          update_theta_a(UpperCAmelCase__			)

          del theta_a
    return final_pipe


 @staticmethod
 def       _lowercase				(    UpperCAmelCase__							:    int,      UpperCAmelCase__							:    Union[str, Any],      UpperCAmelCase__							:    Tuple,      UpperCAmelCase__							:    List[Any]			):
    return ((1 - alpha) * thetaa) + (alpha * thetaa)


 @staticmethod
 def       _lowercase				(    UpperCAmelCase__							:    Union[str, Any],      UpperCAmelCase__							:    Tuple,      UpperCAmelCase__							:    Tuple,      UpperCAmelCase__							:    Dict			):
    __lowercase			=		alpha * alpha * (3 - (2 * alpha))
    return thetaa + ((thetaa - thetaa) * alpha)


 @staticmethod
 def       _lowercase				(    UpperCAmelCase__							:    Dict,      UpperCAmelCase__							:    Optional[Any],      UpperCAmelCase__							:    Dict,      UpperCAmelCase__							:    Dict			):
    import math

    __lowercase			=		0.5 - math.sin(math.asin(1.0 - 2.0 * alpha			) / 3.0			)
    return thetaa + ((thetaa - thetaa) * alpha)



 @staticmethod
 def       _lowercase				(    UpperCAmelCase__							:    str,      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    Tuple			):
    return thetaa + (thetaa - thetaa) * (1.0 - alpha)







"""simple docstring"""





import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		Tuple:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file.jsonl"
   __lowercase			=		textwrap.dedent(
       "\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		Any:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_utf16_encoded.jsonl"
   __lowercase			=		textwrap.dedent(
       "\\n        {\"col_1\": -1}\n        {\"col_1\": 1, \"col_2\": 2}\n        {\"col_1\": 10, \"col_2\": 20}\n        ")
   with open(UpperCamelCase_,    "w",    encoding="utf-16") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : Optional[int])			->		Optional[Any]:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_with_list_of_dicts.json"
   __lowercase			=		textwrap.dedent(
       "\\n        [\n            {\"col_1\": -1},\n            {\"col_1\": 1, \"col_2\": 2},\n            {\"col_1\": 10, \"col_2\": 20}\n        ]\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.fixture
def 						_A						(						UpperCamelCase_  : str)			->		int:
   '''simple docstring'''


   __lowercase			=		tmp_path / "file_with_list_of_dicts_field.json"
   __lowercase			=		textwrap.dedent(
       "\\n        {\n            \"field1\": 1,\n            \"field2\": \"aabb\",\n            \"field3\": [\n                {\"col_1\": -1},\n                {\"col_1\": 1, \"col_2\": 2},\n                {\"col_1\": 10, \"col_2\": 20}\n            ]\n        }\n        ")
   with open(UpperCamelCase_,    "w") as f:
      f.write(UpperCamelCase_)
   return str(UpperCamelCase_)




@pytest.mark.parametrize(
    "file_fixture, config_kwargs",    [
        ("jsonl_file", {}),
        ("jsonl_file_utf16_encoded", {"encoding": "utf-16"}),
        ("json_file_with_list_of_dicts", {}),
        ("json_file_with_list_of_dicts_field", {"field": "field3"}),
    ],    )
def 						_A						(						UpperCamelCase_  : Any,    UpperCamelCase_  : List[Any],    UpperCamelCase_  : Optional[Any])			->		List[Any]:
   '''simple docstring'''


   __lowercase			=		Json(**UpperCamelCase_)
   __lowercase			=		json._generate_tables([[request.getfixturevalue(UpperCamelCase_)]])
   __lowercase			=		pa.concat_tables([table for _, table in generator])
   assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}




@pytest.mark.parametrize(
    "file_fixture, config_kwargs",    [
        (
            "jsonl_file",
            {"features": Features({"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")})},
        ),
        (
            "json_file_with_list_of_dicts",
            {"features": Features({"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")})},
        ),
        (
            "json_file_with_list_of_dicts_field",
            {
                "field": "field3",
                "features": Features(
                    {"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")}),
            },
        ),
    ],    )
def 						_A						(						UpperCamelCase_  : Optional[int],    UpperCamelCase_  : int,    UpperCamelCase_  : str)			->		Optional[int]:
   '''simple docstring'''


   __lowercase			=		Json(**UpperCamelCase_)
   __lowercase			=		json._generate_tables([[request.getfixturevalue(UpperCamelCase_)]])
   __lowercase			=		pa.concat_tables([table for _, table in generator])
   assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}




