




#
# Sort large text files in a minimum amount of memory
#
import argparse
import os







class 		__UpperCAmelCase:





     """simple docstring"""



     __UpperCAmelCase	: Dict						=    """block_{0}.dat"""

     def __init__(      self :       Optional[Any]						,				A :       List[Any]    )			->			Dict:






         """simple docstring"""
         a__      =       filename
         a__      =       []

     def 				__SCREAMING_SNAKE_CASE		(      self :       int						,				A :       Any						,				A :       int    )			->			List[str]:






         """simple docstring"""
         a__      =       self.BLOCK_FILENAME_FORMAT.format(A    )
         with open(A						,				"""w"""    ) as file:
             file.write(A    )
         self.block_filenames.append(A    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       List[Any]    )			->			Optional[Any]:






         """simple docstring"""
         return self.block_filenames

     def 				__SCREAMING_SNAKE_CASE		(      self :       int						,				A :       List[Any]						,				A :       Union[str, Any]=None    )			->			Union[str, Any]:






         """simple docstring"""
         a__      =       0
         with open(self.filename    ) as file:
             while True:
                 a__      =       file.readlines(A    )

                 if lines == []:
                     break

                 if sort_key is None:
                     lines.sort()
                 else:
                     lines.sort(key=A    )

                 self.write_block("""""".join(A    )						,				A    )
                 i += 1






     def 				__SCREAMING_SNAKE_CASE		(      self :       Tuple    )			->			Dict:






         """simple docstring"""
         map(os.remove						,				self.block_filenames    )







class 		__UpperCAmelCase:





     """simple docstring"""



     def 				__SCREAMING_SNAKE_CASE		(      self :       Dict						,				A :       List[str]    )			->			Union[str, Any]:






         """simple docstring"""
         a__      =       -1
         a__      =       None

         for i in range(len(A    )    ):
             if min_str is None or choices[i] < min_str:
                 a__      =       i

         return min_index







class 		__UpperCAmelCase:





     """simple docstring"""



     def __init__(      self :       Union[str, Any]						,				A :       List[str]    )			->			str:






         """simple docstring"""
         a__      =       files
         a__      =       set()
         a__      =       len(A    )
         a__      =       {i: None for i in range(self.num_buffers    )}

     def 				__SCREAMING_SNAKE_CASE		(      self :       str    )			->			str:






         """simple docstring"""
         return {
             i: self.buffers[i] for i in range(self.num_buffers    ) if i not in self.empty
         }

     def 				__SCREAMING_SNAKE_CASE		(      self :       Optional[int]    )			->			Any:






         """simple docstring"""
         for i in range(self.num_buffers    ):
             if self.buffers[i] is None and i not in self.empty:
                 a__      =       self.files[i].readline()

                 if self.buffers[i] == "":
                     self.empty.add(A    )
                     self.files[i].close()

         if len(self.empty    ) == self.num_buffers:
             return False

         return True






     def 				__SCREAMING_SNAKE_CASE		(      self :       Tuple						,				A :       Any    )			->			Tuple:






         """simple docstring"""
         a__      =       self.buffers[index]
         a__      =       None

         return value







class 		__UpperCAmelCase:





     """simple docstring"""



     def __init__(      self :       List[Any]						,				A :       Any    )			->			List[Any]:






         """simple docstring"""
         a__      =       merge_strategy

     def 				__SCREAMING_SNAKE_CASE		(      self :       List[str]						,				A :       int						,				A :       Optional[int]						,				A :       List[Any]    )			->			List[str]:






         """simple docstring"""
         a__      =       FilesArray(self.get_file_handles(A						,				A    )    )
         with open(A						,				"""w"""						,				A    ) as outfile:
             while buffers.refresh():
                 a__      =       self.merge_strategy.select(buffers.get_dict()    )
                 outfile.write(buffers.unshift(A    )    )






     def 				__SCREAMING_SNAKE_CASE		(      self :       int						,				A :       Dict						,				A :       str    )			->			List[Any]:






         """simple docstring"""
         a__      =       {}

         for i in range(len(A    )    ):
             a__      =       open(filenames[i]						,				"""r"""						,				A    )  # noqa: UP015

         return files







class 		__UpperCAmelCase:





     """simple docstring"""



     def __init__(      self :       Dict						,				A :       Union[str, Any]    )			->			Dict:






         """simple docstring"""
         a__      =       block_size

     def 				__SCREAMING_SNAKE_CASE		(      self :       Any						,				A :       Optional[int]						,				A :       Any=None    )			->			str:






         """simple docstring"""
         a__      =       self.get_number_blocks(A						,				self.block_size    )
         a__      =       FileSplitter(A    )
         splitter.split(self.block_size						,				A    )

         a__      =       FileMerger(NWayMerge()    )
         a__      =       self.block_size / (num_blocks + 1)
         merger.merge(splitter.get_block_filenames()						,				filename + """.out"""						,				A    )

         splitter.cleanup()






     def 				__SCREAMING_SNAKE_CASE		(      self :       Tuple						,				A :       List[Any]						,				A :       Optional[Any]    )			->			Dict:






         """simple docstring"""
         return (os.stat(A    ).st_size / block_size) + 1






def        __lowerCAmelCase (UpperCAmelCase_		):
    if string[-1].lower() == "k":
        return int(string[:-1]		) * 1_024
    elif string[-1].lower() == "m":
        return int(string[:-1]		) * 1_024 * 1_024
    elif string[-1].lower() == "g":
        return int(string[:-1]		) * 1_024 * 1_024 * 1_024
    else:
        return int(UpperCAmelCase_		)
def        __lowerCAmelCase ():
    a__      =       argparse.ArgumentParser()
    parser.add_argument(
        """-m"""   ,    """--mem"""   ,    help="""amount of memory to use for sorting"""   ,    default="""100M"""		)
    parser.add_argument(
        """filename"""   ,    metavar="""<filename>"""   ,    nargs=1   ,    help="""name of file to sort"""		)
    a__      =       parser.parse_args()

    a__      =       ExternalSort(parse_memory(args.mem		)		)
    sorter.sort(args.filename[0]		)


if __name__ == "__main__":
    main()












import argparse
import os

import torch
from torchvision.datasets.utils import download_url

from diffusers import AutoencoderKL, DDIMScheduler, DiTPipeline, TransformeraDModel


SCREAMING_SNAKE_CASE_						:					Tuple			   =       {5_12: '''DiT-XL-2-512x512.pt''', 2_56: '''DiT-XL-2-256x256.pt'''}
def        __lowerCAmelCase (UpperCAmelCase_		):
    a__      =       F"""pretrained_models/{model_name}"""
    if not os.path.isfile(UpperCAmelCase_		):
        os.makedirs("""pretrained_models"""   ,    exist_ok=UpperCAmelCase_		)
        a__      =       F"""https://dl.fbaipublicfiles.com/DiT/models/{model_name}"""
        download_url(UpperCAmelCase_   ,    """pretrained_models"""		)
    a__      =       torch.load(UpperCAmelCase_   ,    map_location=lambda UpperCAmelCase_   ,    UpperCAmelCase_		: storage		)
    return model
def        __lowerCAmelCase (UpperCAmelCase_		):
    a__      =       download_model(pretrained_models[args.image_size]		)

    a__      =       state_dict["""x_embedder.proj.weight"""]
    a__      =       state_dict["""x_embedder.proj.bias"""]
    state_dict.pop("""x_embedder.proj.weight"""		)
    state_dict.pop("""x_embedder.proj.bias"""		)

    for depth in range(28		):
        a__      =       state_dict[
            """t_embedder.mlp.0.weight"""
        ]
        a__      =       state_dict[
            """t_embedder.mlp.0.bias"""
        ]
        a__      =       state_dict[
            """t_embedder.mlp.2.weight"""
        ]
        a__      =       state_dict[
            """t_embedder.mlp.2.bias"""
        ]
        a__      =       state_dict[
            """y_embedder.embedding_table.weight"""
        ]

        a__      =       state_dict[
            F"""blocks.{depth}.adaLN_modulation.1.weight"""
        ]
        a__      =       state_dict[
            F"""blocks.{depth}.adaLN_modulation.1.bias"""
        ]

        a__ ,		a__ ,		a__      =       torch.chunk(state_dict[F"""blocks.{depth}.attn.qkv.weight"""]   ,    3   ,    dim=0		)
        a__ ,		a__ ,		a__      =       torch.chunk(state_dict[F"""blocks.{depth}.attn.qkv.bias"""]   ,    3   ,    dim=0		)

        a__      =       q
        a__      =       q_bias
        a__      =       k
        a__      =       k_bias
        a__      =       v
        a__      =       v_bias

        a__      =       state_dict[
            F"""blocks.{depth}.attn.proj.weight"""
        ]
        a__      =       state_dict[F"""blocks.{depth}.attn.proj.bias"""]

        a__      =       state_dict[F"""blocks.{depth}.mlp.fc1.weight"""]
        a__      =       state_dict[F"""blocks.{depth}.mlp.fc1.bias"""]
        a__      =       state_dict[F"""blocks.{depth}.mlp.fc2.weight"""]
        a__      =       state_dict[F"""blocks.{depth}.mlp.fc2.bias"""]

        state_dict.pop(F"""blocks.{depth}.attn.qkv.weight"""		)
        state_dict.pop(F"""blocks.{depth}.attn.qkv.bias"""		)
        state_dict.pop(F"""blocks.{depth}.attn.proj.weight"""		)
        state_dict.pop(F"""blocks.{depth}.attn.proj.bias"""		)
        state_dict.pop(F"""blocks.{depth}.mlp.fc1.weight"""		)
        state_dict.pop(F"""blocks.{depth}.mlp.fc1.bias"""		)
        state_dict.pop(F"""blocks.{depth}.mlp.fc2.weight"""		)
        state_dict.pop(F"""blocks.{depth}.mlp.fc2.bias"""		)
        state_dict.pop(F"""blocks.{depth}.adaLN_modulation.1.weight"""		)
        state_dict.pop(F"""blocks.{depth}.adaLN_modulation.1.bias"""		)

    state_dict.pop("""t_embedder.mlp.0.weight"""		)
    state_dict.pop("""t_embedder.mlp.0.bias"""		)
    state_dict.pop("""t_embedder.mlp.2.weight"""		)
    state_dict.pop("""t_embedder.mlp.2.bias"""		)
    state_dict.pop("""y_embedder.embedding_table.weight"""		)

    a__      =       state_dict["""final_layer.adaLN_modulation.1.weight"""]
    a__      =       state_dict["""final_layer.adaLN_modulation.1.bias"""]
    a__      =       state_dict["""final_layer.linear.weight"""]
    a__      =       state_dict["""final_layer.linear.bias"""]

    state_dict.pop("""final_layer.linear.weight"""		)
    state_dict.pop("""final_layer.linear.bias"""		)
    state_dict.pop("""final_layer.adaLN_modulation.1.weight"""		)
    state_dict.pop("""final_layer.adaLN_modulation.1.bias"""		)

    # DiT XL/2
    a__      =       TransformeraDModel(
        sample_size=args.image_size // 8   ,    num_layers=28   ,    attention_head_dim=72   ,    in_channels=4   ,    out_channels=8   ,    patch_size=2   ,    attention_bias=UpperCAmelCase_   ,    num_attention_heads=16   ,    activation_fn="""gelu-approximate"""   ,    num_embeds_ada_norm=1_000   ,    norm_type="""ada_norm_zero"""   ,    norm_elementwise_affine=UpperCAmelCase_   ,    )
    transformer.load_state_dict(UpperCAmelCase_   ,    strict=UpperCAmelCase_		)

    a__      =       DDIMScheduler(
        num_train_timesteps=1_000   ,    beta_schedule="""linear"""   ,    prediction_type="""epsilon"""   ,    clip_sample=UpperCAmelCase_   ,    )

    a__      =       AutoencoderKL.from_pretrained(args.vae_model		)

    a__      =       DiTPipeline(transformer=UpperCAmelCase_   ,    vae=UpperCAmelCase_   ,    scheduler=UpperCAmelCase_		)

    if args.save:
        pipeline.save_pretrained(args.checkpoint_path		)


if __name__ == "__main__":
 SCREAMING_SNAKE_CASE_						:					Optional[Any]			   =       argparse.ArgumentParser()

 parser.add_argument(
     '''--image_size''',
     default=2_56,
     type=int,
     required=False,
     help='''Image size of pretrained model, either 256 or 512.''',
 )
 parser.add_argument(
     '''--vae_model''',
     default='''stabilityai/sd-vae-ft-ema''',
     type=str,
     required=False,
     help='''Path to pretrained VAE model, either stabilityai/sd-vae-ft-mse or stabilityai/sd-vae-ft-ema.''',
 )
 parser.add_argument(
     '''--save''', default=True, type=bool, required=False, help='''Whether to save the converted pipeline or not.'''
 )
 parser.add_argument(
     '''--checkpoint_path''', default=None, type=str, required=True, help='''Path to the output pipeline.'''
 )

 SCREAMING_SNAKE_CASE_						:					Optional[int]			   =       parser.parse_args()
 main(args)






