






#
# Sort large text files in a minimum amount of memory
#
import argparse
import os



class 				__snake_case  :


    '''simple docstring'''







    lowerCamelCase__						:		Union[str, Any]							=  """block_{0}.dat"""



    def __init__(					self      ,							A_    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= filename
        SCREAMING_SNAKE_CASE__       							= []



    def    lowercase_				(					self      ,							A_      ,							A_    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= self.BLOCK_FILENAME_FORMAT.format(A_    )
        with open(A_      ,							'''w'''    ) as file:
            file.write(A_    )
        self.block_filenames.append(A_    )



    def    lowercase_				(					self    ):





        '''simple docstring'''

        return self.block_filenames



    def    lowercase_				(					self      ,							A_      ,							A_=None    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= 0
        with open(self.filename    ) as file:
            while True:
                SCREAMING_SNAKE_CASE__       							= file.readlines(A_    )

                if lines == []:
                    break

                if sort_key is None:
                    lines.sort()
                else:
                    lines.sort(key=A_    )

                self.write_block(''''''.join(A_    )      ,							A_    )
                i += 1




    def    lowercase_				(					self    ):





        '''simple docstring'''

        map(os.remove      ,							self.block_filenames    )





class 				__snake_case  :


    '''simple docstring'''







    def    lowercase_				(					self      ,							A_    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= -1
        SCREAMING_SNAKE_CASE__       							= None

        for i in range(len(A_    )    ):
            if min_str is None or choices[i] < min_str:
                SCREAMING_SNAKE_CASE__       							= i

        return min_index





class 				__snake_case  :


    '''simple docstring'''







    def __init__(					self      ,							A_    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= files
        SCREAMING_SNAKE_CASE__       							= set()
        SCREAMING_SNAKE_CASE__       							= len(A_    )
        SCREAMING_SNAKE_CASE__       							= {i: None for i in range(self.num_buffers    )}



    def    lowercase_				(					self    ):





        '''simple docstring'''

        return {
            i: self.buffers[i] for i in range(self.num_buffers    ) if i not in self.empty
        }



    def    lowercase_				(					self    ):





        '''simple docstring'''

        for i in range(self.num_buffers    ):
            if self.buffers[i] is None and i not in self.empty:
                SCREAMING_SNAKE_CASE__       							= self.files[i].readline()

                if self.buffers[i] == "":
                    self.empty.add(A_    )
                    self.files[i].close()

        if len(self.empty    ) == self.num_buffers:
            return False

        return True




    def    lowercase_				(					self      ,							A_    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= self.buffers[index]
        SCREAMING_SNAKE_CASE__       							= None

        return value





class 				__snake_case  :


    '''simple docstring'''







    def __init__(					self      ,							A_    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= merge_strategy



    def    lowercase_				(					self      ,							A_      ,							A_      ,							A_    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= FilesArray(self.get_file_handles(A_      ,							A_    )    )
        with open(A_      ,							'''w'''      ,							A_    ) as outfile:
            while buffers.refresh():
                SCREAMING_SNAKE_CASE__       							= self.merge_strategy.select(buffers.get_dict()    )
                outfile.write(buffers.unshift(A_    )    )




    def    lowercase_				(					self      ,							A_      ,							A_    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= {}

        for i in range(len(A_    )    ):
            SCREAMING_SNAKE_CASE__       							= open(filenames[i]      ,							'''r'''      ,							A_    )  # noqa: UP015

        return files





class 				__snake_case  :


    '''simple docstring'''







    def __init__(					self      ,							A_    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= block_size



    def    lowercase_				(					self      ,							A_      ,							A_=None    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= self.get_number_blocks(A_      ,							self.block_size    )
        SCREAMING_SNAKE_CASE__       							= FileSplitter(A_    )
        splitter.split(self.block_size      ,							A_    )

        SCREAMING_SNAKE_CASE__       							= FileMerger(NWayMerge()    )
        SCREAMING_SNAKE_CASE__       							= self.block_size / (num_blocks + 1)
        merger.merge(splitter.get_block_filenames()      ,							filename + '''.out'''      ,							A_    )

        splitter.cleanup()




    def    lowercase_				(					self      ,							A_      ,							A_    ):





        '''simple docstring'''

        return (os.stat(A_    ).st_size / block_size) + 1





def        __snake_case       (						lowerCAmelCase_  )							->	Any:
    if string[-1].lower() == "k":
        return int(string[:-1]  ) * 1_0_2_4
    elif string[-1].lower() == "m":
        return int(string[:-1]  ) * 1_0_2_4 * 1_0_2_4
    elif string[-1].lower() == "g":
        return int(string[:-1]  ) * 1_0_2_4 * 1_0_2_4 * 1_0_2_4
    else:
        return int(lowerCAmelCase_  )


def        __snake_case       (						)							->	List[str]:
    SCREAMING_SNAKE_CASE__       							= argparse.ArgumentParser()
    parser.add_argument(
        '''-m'''   ,       '''--mem'''   ,       help='''amount of memory to use for sorting'''   ,       default='''100M'''  )
    parser.add_argument(
        '''filename'''   ,       metavar='''<filename>'''   ,       nargs=1   ,       help='''name of file to sort'''  )
    SCREAMING_SNAKE_CASE__       							= parser.parse_args()

    SCREAMING_SNAKE_CASE__       							= ExternalSort(parse_memory(args.mem  )  )
    sorter.sort(args.filename[0]  )


if __name__ == "__main__":
  main()







import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
  import transformer_engine.pytorch as te


def        __snake_case       (						lowerCAmelCase_   ,       lowerCAmelCase_=True   ,       lowerCAmelCase_=True   ,       lowerCAmelCase_=True  )							->	Tuple:
    if not is_fpa_available():
        raise ImportError('''Using `convert_model` requires transformer_engine to be installed.'''  )
    for name, module in model.named_children():
        if isinstance(lowerCAmelCase_   ,       nn.Linear  ) and to_transformer_engine and _convert_linear:
            # Return early if the linear layer weights are not multiples of 16
            if any(p % 1_6 != 0 for p in module.weight.shape  ):
                return
            SCREAMING_SNAKE_CASE__       							= module.bias is not None
            SCREAMING_SNAKE_CASE__       							= te.Linear(
                module.in_features   ,       module.out_features   ,       bias=lowerCAmelCase_   ,       params_dtype=module.weight.dtype  )
            SCREAMING_SNAKE_CASE__       							= module.weight.data.clone()
            if has_bias:
                SCREAMING_SNAKE_CASE__       							= module.bias.data.clone()

            setattr(lowerCAmelCase_   ,       lowerCAmelCase_   ,       lowerCAmelCase_  )
        elif isinstance(lowerCAmelCase_   ,       nn.LayerNorm  ) and to_transformer_engine and _convert_ln:
            SCREAMING_SNAKE_CASE__       							= te.LayerNorm(module.normalized_shape[0]   ,       eps=module.eps   ,       params_dtype=module.weight.dtype  )
            SCREAMING_SNAKE_CASE__       							= module.weight.data.clone()
            SCREAMING_SNAKE_CASE__       							= module.bias.data.clone()

            setattr(lowerCAmelCase_   ,       lowerCAmelCase_   ,       lowerCAmelCase_  )
        elif isinstance(lowerCAmelCase_   ,       te.Linear  ) and not to_transformer_engine and _convert_linear:
            SCREAMING_SNAKE_CASE__       							= module.bias is not None
            SCREAMING_SNAKE_CASE__       							= nn.Linear(
                module.in_features   ,       module.out_features   ,       bias=lowerCAmelCase_   ,       params_dtype=module.weight.dtype  )
            SCREAMING_SNAKE_CASE__       							= module.weight.data.clone()
            if has_bias:
                SCREAMING_SNAKE_CASE__       							= module.bias.data.clone()

            setattr(lowerCAmelCase_   ,       lowerCAmelCase_   ,       lowerCAmelCase_  )
        elif isinstance(lowerCAmelCase_   ,       te.LayerNorm  ) and not to_transformer_engine and _convert_ln:
            SCREAMING_SNAKE_CASE__       							= nn.LayerNorm(module.normalized_shape[0]   ,       eps=module.eps   ,       params_dtype=module.weight.dtype  )
            SCREAMING_SNAKE_CASE__       							= module.weight.data.clone()
            SCREAMING_SNAKE_CASE__       							= module.bias.data.clone()

            setattr(lowerCAmelCase_   ,       lowerCAmelCase_   ,       lowerCAmelCase_  )
        else:
            convert_model(
                lowerCAmelCase_   ,       to_transformer_engine=lowerCAmelCase_   ,       _convert_linear=_convert_linear   ,       _convert_ln=_convert_ln   ,       )



def        __snake_case       (						lowerCAmelCase_  )							->	Union[str, Any]:
    if not is_fpa_available():
        raise ImportError('''Using `has_transformer_engine_layers` requires transformer_engine to be installed.'''  )
    for m in model.modules():
        if isinstance(lowerCAmelCase_   ,       (te.LayerNorm, te.Linear)  ):
            return True
    return False
