




#
# Sort large text files in a minimum amount of memory
#
import argparse
import os







class 		__UpperCAmelCase:





     """simple docstring"""



     __UpperCAmelCase	: Dict						=    """block_{0}.dat"""

     def __init__(      self :       Optional[Any]						,				A :       List[Any]    )			->			Dict:






         """simple docstring"""
         a__      =       filename
         a__      =       []

     def 				__SCREAMING_SNAKE_CASE		(      self :       int						,				A :       Any						,				A :       int    )			->			List[str]:






         """simple docstring"""
         a__      =       self.BLOCK_FILENAME_FORMAT.format(A    )
         with open(A						,				"""w"""    ) as file:
             file.write(A    )
         self.block_filenames.append(A    )

     def 				__SCREAMING_SNAKE_CASE		(      self :       List[Any]    )			->			Optional[Any]:






         """simple docstring"""
         return self.block_filenames

     def 				__SCREAMING_SNAKE_CASE		(      self :       int						,				A :       List[Any]						,				A :       Union[str, Any]=None    )			->			Union[str, Any]:






         """simple docstring"""
         a__      =       0
         with open(self.filename    ) as file:
             while True:
                 a__      =       file.readlines(A    )

                 if lines == []:
                     break

                 if sort_key is None:
                     lines.sort()
                 else:
                     lines.sort(key=A    )

                 self.write_block("""""".join(A    )						,				A    )
                 i += 1






     def 				__SCREAMING_SNAKE_CASE		(      self :       Tuple    )			->			Dict:






         """simple docstring"""
         map(os.remove						,				self.block_filenames    )







class 		__UpperCAmelCase:





     """simple docstring"""



     def 				__SCREAMING_SNAKE_CASE		(      self :       Dict						,				A :       List[str]    )			->			Union[str, Any]:






         """simple docstring"""
         a__      =       -1
         a__      =       None

         for i in range(len(A    )    ):
             if min_str is None or choices[i] < min_str:
                 a__      =       i

         return min_index







class 		__UpperCAmelCase:





     """simple docstring"""



     def __init__(      self :       Union[str, Any]						,				A :       List[str]    )			->			str:






         """simple docstring"""
         a__      =       files
         a__      =       set()
         a__      =       len(A    )
         a__      =       {i: None for i in range(self.num_buffers    )}

     def 				__SCREAMING_SNAKE_CASE		(      self :       str    )			->			str:






         """simple docstring"""
         return {
             i: self.buffers[i] for i in range(self.num_buffers    ) if i not in self.empty
         }

     def 				__SCREAMING_SNAKE_CASE		(      self :       Optional[int]    )			->			Any:






         """simple docstring"""
         for i in range(self.num_buffers    ):
             if self.buffers[i] is None and i not in self.empty:
                 a__      =       self.files[i].readline()

                 if self.buffers[i] == "":
                     self.empty.add(A    )
                     self.files[i].close()

         if len(self.empty    ) == self.num_buffers:
             return False

         return True






     def 				__SCREAMING_SNAKE_CASE		(      self :       Tuple						,				A :       Any    )			->			Tuple:






         """simple docstring"""
         a__      =       self.buffers[index]
         a__      =       None

         return value







class 		__UpperCAmelCase:





     """simple docstring"""



     def __init__(      self :       List[Any]						,				A :       Any    )			->			List[Any]:






         """simple docstring"""
         a__      =       merge_strategy

     def 				__SCREAMING_SNAKE_CASE		(      self :       List[str]						,				A :       int						,				A :       Optional[int]						,				A :       List[Any]    )			->			List[str]:






         """simple docstring"""
         a__      =       FilesArray(self.get_file_handles(A						,				A    )    )
         with open(A						,				"""w"""						,				A    ) as outfile:
             while buffers.refresh():
                 a__      =       self.merge_strategy.select(buffers.get_dict()    )
                 outfile.write(buffers.unshift(A    )    )






     def 				__SCREAMING_SNAKE_CASE		(      self :       int						,				A :       Dict						,				A :       str    )			->			List[Any]:






         """simple docstring"""
         a__      =       {}

         for i in range(len(A    )    ):
             a__      =       open(filenames[i]						,				"""r"""						,				A    )  # noqa: UP015

         return files







class 		__UpperCAmelCase:





     """simple docstring"""



     def __init__(      self :       Dict						,				A :       Union[str, Any]    )			->			Dict:






         """simple docstring"""
         a__      =       block_size

     def 				__SCREAMING_SNAKE_CASE		(      self :       Any						,				A :       Optional[int]						,				A :       Any=None    )			->			str:






         """simple docstring"""
         a__      =       self.get_number_blocks(A						,				self.block_size    )
         a__      =       FileSplitter(A    )
         splitter.split(self.block_size						,				A    )

         a__      =       FileMerger(NWayMerge()    )
         a__      =       self.block_size / (num_blocks + 1)
         merger.merge(splitter.get_block_filenames()						,				filename + """.out"""						,				A    )

         splitter.cleanup()






     def 				__SCREAMING_SNAKE_CASE		(      self :       Tuple						,				A :       List[Any]						,				A :       Optional[Any]    )			->			Dict:






         """simple docstring"""
         return (os.stat(A    ).st_size / block_size) + 1






def        __lowerCAmelCase (UpperCAmelCase_		):
    if string[-1].lower() == "k":
        return int(string[:-1]		) * 1_024
    elif string[-1].lower() == "m":
        return int(string[:-1]		) * 1_024 * 1_024
    elif string[-1].lower() == "g":
        return int(string[:-1]		) * 1_024 * 1_024 * 1_024
    else:
        return int(UpperCAmelCase_		)
def        __lowerCAmelCase ():
    a__      =       argparse.ArgumentParser()
    parser.add_argument(
        """-m"""   ,    """--mem"""   ,    help="""amount of memory to use for sorting"""   ,    default="""100M"""		)
    parser.add_argument(
        """filename"""   ,    metavar="""<filename>"""   ,    nargs=1   ,    help="""name of file to sort"""		)
    a__      =       parser.parse_args()

    a__      =       ExternalSort(parse_memory(args.mem		)		)
    sorter.sort(args.filename[0]		)


if __name__ == "__main__":
 main()












import unittest

from transformers import (
    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
    PreTrainedTokenizer,
    is_vision_available,
)
from transformers.pipelines import ImageClassificationPipeline, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_or_tf,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
 from PIL import Image
else:







 class 		__UpperCAmelCase:





      """simple docstring"""



      @staticmethod
      def 				__SCREAMING_SNAKE_CASE		(      *A :       str						,				**A :       int    )			->			Union[str, Any]:






          """simple docstring"""
          pass







@is_pipeline_test
@require_torch_or_tf
@require_vision
class 		__UpperCAmelCase(		unittest.TestCase ):





     """simple docstring"""



     __UpperCAmelCase	: Tuple						=    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
     __UpperCAmelCase	: Dict						=    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING

     def 				__SCREAMING_SNAKE_CASE		(      self :       str						,				A :       Optional[int]						,				A :       Tuple						,				A :       int    )			->			str:






         """simple docstring"""
         a__      =       ImageClassificationPipeline(model=A						,				image_processor=A						,				top_k=2    )
         a__      =       [
             Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""    ),
             """http://images.cocodataset.org/val2017/000000039769.jpg""",
         ]
         return image_classifier, examples

     def 				__SCREAMING_SNAKE_CASE		(      self :       Any						,				A :       List[str]						,				A :       Dict    )			->			Union[str, Any]:






         """simple docstring"""
         a__      =       image_classifier("""./tests/fixtures/tests_samples/COCO/000000039769.png"""    )

         self.assertEqual(
             A						,				[
                 {"""score""": ANY(A    ), """label""": ANY(A    )},
                 {"""score""": ANY(A    ), """label""": ANY(A    )},
             ]						,				)

         import datasets

         a__      =       datasets.load_dataset("""hf-internal-testing/fixtures_image_utils"""						,				"""image"""						,				split="""test"""    )

         # Accepts URL + PIL.Image + lists
         a__      =       image_classifier(
             [
                 Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""    ),
                 """http://images.cocodataset.org/val2017/000000039769.jpg""",
                 # RGBA
                 dataset[0]["""file"""],
                 # LA
                 dataset[1]["""file"""],
                 # L
                 dataset[2]["""file"""],
             ]    )
         self.assertEqual(
             A						,				[
                 [
                     {"""score""": ANY(A    ), """label""": ANY(A    )},
                     {"""score""": ANY(A    ), """label""": ANY(A    )},
                 ],
                 [
                     {"""score""": ANY(A    ), """label""": ANY(A    )},
                     {"""score""": ANY(A    ), """label""": ANY(A    )},
                 ],
                 [
                     {"""score""": ANY(A    ), """label""": ANY(A    )},
                     {"""score""": ANY(A    ), """label""": ANY(A    )},
                 ],
                 [
                     {"""score""": ANY(A    ), """label""": ANY(A    )},
                     {"""score""": ANY(A    ), """label""": ANY(A    )},
                 ],
                 [
                     {"""score""": ANY(A    ), """label""": ANY(A    )},
                     {"""score""": ANY(A    ), """label""": ANY(A    )},
                 ],
             ]						,				)

     @require_torch
     def 				__SCREAMING_SNAKE_CASE		(      self :       Optional[int]    )			->			Optional[Any]:






         """simple docstring"""
         a__      =       """hf-internal-testing/tiny-random-vit"""
         a__      =       pipeline("""image-classification"""						,				model=A    )

         a__      =       image_classifier("""http://images.cocodataset.org/val2017/000000039769.jpg"""    )
         self.assertEqual(
             nested_simplify(A						,				decimals=4    )						,				[{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}]						,				)

         a__      =       image_classifier(
             [
                 """http://images.cocodataset.org/val2017/000000039769.jpg""",
                 """http://images.cocodataset.org/val2017/000000039769.jpg""",
             ]						,				top_k=2						,				)
         self.assertEqual(
             nested_simplify(A						,				decimals=4    )						,				[
                 [{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}],
                 [{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}],
             ]						,				)

     @require_tf
     def 				__SCREAMING_SNAKE_CASE		(      self :       Dict    )			->			str:






         """simple docstring"""
         a__      =       """hf-internal-testing/tiny-random-vit"""
         a__      =       pipeline("""image-classification"""						,				model=A						,				framework="""tf"""    )

         a__      =       image_classifier("""http://images.cocodataset.org/val2017/000000039769.jpg"""    )
         self.assertEqual(
             nested_simplify(A						,				decimals=4    )						,				[{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}]						,				)

         a__      =       image_classifier(
             [
                 """http://images.cocodataset.org/val2017/000000039769.jpg""",
                 """http://images.cocodataset.org/val2017/000000039769.jpg""",
             ]						,				top_k=2						,				)
         self.assertEqual(
             nested_simplify(A						,				decimals=4    )						,				[
                 [{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}],
                 [{"""label""": """LABEL_1""", """score""": 0.5_7_4}, {"""label""": """LABEL_0""", """score""": 0.4_2_6}],
             ]						,				)

     def 				__SCREAMING_SNAKE_CASE		(      self :       Union[str, Any]    )			->			str:






         """simple docstring"""
         a__      =       PreTrainedTokenizer()

         # Assert that the pipeline can be initialized with a feature extractor that is not in any mapping
         a__      =       pipeline(
             """image-classification"""						,				model="""hf-internal-testing/tiny-random-vit"""						,				tokenizer=A    )

         self.assertIs(image_classifier.tokenizer						,				A    )






     @slow
     @require_torch
     def 				__SCREAMING_SNAKE_CASE		(      self :       Optional[Any]    )			->			Dict:






         """simple docstring"""
         # Perceiver is not tested by `run_pipeline_test` properly.
         # That is because the type of feature_extractor and model preprocessor need to be kept
         # in sync, which is not the case in the current design
         a__      =       pipeline("""image-classification"""						,				model="""deepmind/vision-perceiver-conv"""    )
         a__      =       image_classifier("""http://images.cocodataset.org/val2017/000000039769.jpg"""    )
         self.assertEqual(
             nested_simplify(A						,				decimals=4    )						,				[
                 {"""score""": 0.4_3_8_5, """label""": """tabby, tabby cat"""},
                 {"""score""": 0.3_2_1, """label""": """tiger cat"""},
                 {"""score""": 0.0_5_0_2, """label""": """Egyptian cat"""},
                 {"""score""": 0.0_1_3_7, """label""": """crib, cot"""},
                 {"""score""": 0.0_0_7, """label""": """radiator"""},
             ]						,				)

         a__      =       pipeline("""image-classification"""						,				model="""deepmind/vision-perceiver-fourier"""    )
         a__      =       image_classifier("""http://images.cocodataset.org/val2017/000000039769.jpg"""    )
         self.assertEqual(
             nested_simplify(A						,				decimals=4    )						,				[
                 {"""score""": 0.5_6_5_8, """label""": """tabby, tabby cat"""},
                 {"""score""": 0.1_3_0_9, """label""": """tiger cat"""},
                 {"""score""": 0.0_7_2_2, """label""": """Egyptian cat"""},
                 {"""score""": 0.0_7_0_7, """label""": """remote control, remote"""},
                 {"""score""": 0.0_0_8_2, """label""": """computer keyboard, keypad"""},
             ]						,				)

         a__      =       pipeline("""image-classification"""						,				model="""deepmind/vision-perceiver-learned"""    )
         a__      =       image_classifier("""http://images.cocodataset.org/val2017/000000039769.jpg"""    )
         self.assertEqual(
             nested_simplify(A						,				decimals=4    )						,				[
                 {"""score""": 0.3_0_2_2, """label""": """tabby, tabby cat"""},
                 {"""score""": 0.2_3_6_2, """label""": """Egyptian cat"""},
                 {"""score""": 0.1_8_5_6, """label""": """tiger cat"""},
                 {"""score""": 0.0_3_2_4, """label""": """remote control, remote"""},
                 {"""score""": 0.0_0_9_6, """label""": """quilt, comforter, comfort, puff"""},
             ]						,				)






