




'''simple docstring'''
import json
import os
import subprocess
import sys
import warnings
from argparse import ArgumentParser
from contextlib import AbstractContextManager
from typing import Dict, List, Optional

import requests

from ..utils import logging
from . import BaseTransformersCLICommand


_SCREAMING_SNAKE_CASE =	logging.get_logger(__name__)  # pylint: disable=invalid-name


_SCREAMING_SNAKE_CASE =	"lfs-multipart-upload"
class      __UpperCAmelCase							(    lowerCAmelCase							):



					'''simple docstring'''





					@staticmethod
					def 						__snake_case	(  _lowercase    :	ArgumentParser)						->       int:
								A_       		=   parser.add_parser(
								    'lfs-enable-largefiles'			,					help=(
								        'Deprecated: use `huggingface-cli` instead. Configure your repository to enable upload of files > 5GB.'
								    )			,					)
								enable_parser.add_argument('path'			,					type=_lowercase			,					help='Local path to repository you want to configure.')
								enable_parser.set_defaults(func=lambda _lowercase: LfsEnableCommand(_lowercase))

								A_       		=   parser.add_parser(
								    _lowercase			,					help=(
								        'Deprecated: use `huggingface-cli` instead. '
								        'Command will get called by git-lfs, do not call it directly.'
								    )			,					)
								upload_parser.set_defaults(func=lambda _lowercase: LfsUploadCommand(_lowercase))





class      __UpperCAmelCase							:



					'''simple docstring'''





					def __init__(  self    :	str			,					_lowercase    :	List[str])						->       Optional[int]:
								A_       		=   args



					def 						__snake_case	(  self    :	Tuple)						->       Optional[int]:
								warnings.warn(
								    'Managing repositories through transformers-cli is deprecated. Please use `huggingface-cli` instead.')
								A_       		=   os.path.abspath(self.args.path)
								if not os.path.isdir(_lowercase):
											print('This does not look like a valid git repo.')
											exit(1)
								subprocess.run(
								    'git config lfs.customtransfer.multipart.path transformers-cli'.split()			,					check=_lowercase			,					cwd=_lowercase)
								subprocess.run(
								    F'git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}'.split()			,					check=_lowercase			,					cwd=_lowercase			,					)
								print('Local repo set up for largefiles')





def  lowerCamelCase(       SCREAMING_SNAKE_CASE_    )		->		int:
			A_       		=   json.dumps(SCREAMING_SNAKE_CASE_    ) + '\n'
			sys.stdout.write(SCREAMING_SNAKE_CASE_    )
			sys.stdout.flush()





def  lowerCamelCase(       )		->		Optional[Dict]:
			A_       		=   json.loads(sys.stdin.readline().strip()    )

			if "terminate" in (msg.get('type'    ), msg.get('event'    )):
						# terminate message received
						return None

			if msg.get('event'    ) not in ("download", "upload"):
						logger.critical('Received unexpected message'    )
						sys.exit(1    )

			return msg
class      __UpperCAmelCase							(    lowerCAmelCase							):



					'''simple docstring'''





					def __init__(  self    :	Union[str, Any]			,					_lowercase    :	str			,					_lowercase    :	int			,					_lowercase    :	int)						->       Any:
								A_       		=   filepath
								A_       		=   seek_from
								A_       		=   read_limit
								A_       		=   0



					def __enter__(  self    :	int)						->       Optional[Any]:
								A_       		=   open(self.filepath			,					'rb')
								self.f.seek(self.seek_from)
								return self



					def __len__(  self    :	Optional[Any])						->       Tuple:
								A_       		=   os.fstat(self.f.fileno()).st_size
								return min(self.read_limit			,					total_length - self.seek_from)



					def 						__snake_case	(  self    :	Dict			,					_lowercase    :	str=-1)						->       str:
								if self.n_seen >= self.read_limit:
											return b""
								A_       		=   self.read_limit - self.n_seen
								A_       		=   self.f.read(remaining_amount if n < 0 else min(_lowercase			,					_lowercase))
								self.n_seen += len(_lowercase)
								return data



					def __iter__(  self    :	Any)						->       Optional[int]:
								yield self.read(n=4 * 1_024 * 1_024)



					def __exit__(  self    :	Optional[Any]			,					*_lowercase    :	Optional[int])						->       List[Any]:
								self.f.close()





class      __UpperCAmelCase							:



					'''simple docstring'''





					def __init__(  self    :	Optional[int]			,					_lowercase    :	Tuple)						->       Dict:
								A_       		=   args



					def 						__snake_case	(  self    :	Optional[Any])						->       Tuple:
								# Immediately after invoking a custom transfer process, git-lfs
								# sends initiation data to the process over stdin.
								# This tells the process useful information about the configuration.
								A_       		=   json.loads(sys.stdin.readline().strip())
								if not (init_msg.get('event') == "init" and init_msg.get('operation') == "upload"):
											write_msg({'error': {'code': 32, 'message': 'Wrong lfs init operation'}})
											sys.exit(1)

								# The transfer process should use the information it needs from the
								# initiation structure, and also perform any one-off setup tasks it
								# needs to do. It should then respond on stdout with a simple empty
								# confirmation structure, as follows:
								write_msg({})

								# After the initiation exchange, git-lfs will send any number of
								# transfer requests to the stdin of the transfer process, in a serial sequence.
								while True:
											A_       		=   read_msg()
											if msg is None:
														# When all transfers have been processed, git-lfs will send
														# a terminate event to the stdin of the transfer process.
														# On receiving this message the transfer process should
														# clean up and terminate. No response is expected.
														sys.exit(0)

											A_       		=   msg['oid']
											A_       		=   msg['path']
											A_       		=   msg['action']['href']
											A_       		=   msg['action']['header']
											A_       		=   int(header.pop('chunk_size'))
											A_       		=   list(header.values())

											A_       		=   []
											for i, presigned_url in enumerate(_lowercase):
														with FileSlice(_lowercase			,					seek_from=i * chunk_size			,					read_limit=_lowercase) as data:
																	A_       		=   requests.put(_lowercase			,					data=_lowercase)
																	r.raise_for_status()
																	parts.append(
																	    {
																	        'etag': r.headers.get('etag'),
																	        'partNumber': i + 1,
																	    })
																	# In order to support progress reporting while data is uploading / downloading,
																	# the transfer process should post messages to stdout
																	write_msg(
																	    {
																	        'event': 'progress',
																	        'oid': oid,
																	        'bytesSoFar': (i + 1) * chunk_size,
																	        'bytesSinceLast': chunk_size,
																	    })
																	# Not precise but that's ok.

											A_       		=   requests.post(
											    _lowercase			,					json={
											        'oid': oid,
											        'parts': parts,
											    }			,					)
											r.raise_for_status()

											write_msg({'event': 'complete', 'oid': oid})










'''simple docstring'''
import os
import pickle
import unittest
from collections import UserDict, namedtuple

import torch

from accelerate.test_utils.testing import require_cuda, require_torch_min_version
from accelerate.test_utils.training import RegressionModel
from accelerate.utils import (
    convert_outputs_to_fpaa,
    extract_model_from_parallel,
    find_device,
    listify,
    patch_environment,
    recursively_apply,
    send_to_device,
)


_SCREAMING_SNAKE_CASE =	namedtuple("ExampleNamedTuple", "a b c")
class      __UpperCAmelCase							(    unittest.TestCase							):



					'''simple docstring'''





					def 						__snake_case	(  self    :	List[Any])						->       Tuple:
								A_       		=   torch.randn(5			,					2)
								A_       		=   torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

								A_       		=   send_to_device(_lowercase			,					_lowercase)
								self.assertTrue(torch.equal(resulta.cpu()			,					_lowercase))

								A_       		=   send_to_device((tensor, [tensor, tensor], 1)			,					_lowercase)
								self.assertIsInstance(_lowercase			,					_lowercase)
								self.assertTrue(torch.equal(resulta[0].cpu()			,					_lowercase))
								self.assertIsInstance(resulta[1]			,					_lowercase)
								self.assertTrue(torch.equal(resulta[1][0].cpu()			,					_lowercase))
								self.assertTrue(torch.equal(resulta[1][1].cpu()			,					_lowercase))
								self.assertEqual(resulta[2]			,					1)

								A_       		=   send_to_device({'a': tensor, 'b': [tensor, tensor], 'c': 1}			,					_lowercase)
								self.assertIsInstance(_lowercase			,					_lowercase)
								self.assertTrue(torch.equal(resulta['a'].cpu()			,					_lowercase))
								self.assertIsInstance(resulta['b']			,					_lowercase)
								self.assertTrue(torch.equal(resulta['b'][0].cpu()			,					_lowercase))
								self.assertTrue(torch.equal(resulta['b'][1].cpu()			,					_lowercase))
								self.assertEqual(resulta['c']			,					1)

								A_       		=   send_to_device(ExampleNamedTuple(a=_lowercase			,					b=[tensor, tensor]			,					c=1)			,					_lowercase)
								self.assertIsInstance(_lowercase			,					_lowercase)
								self.assertTrue(torch.equal(resulta.a.cpu()			,					_lowercase))
								self.assertIsInstance(resulta.b			,					_lowercase)
								self.assertTrue(torch.equal(resulta.b[0].cpu()			,					_lowercase))
								self.assertTrue(torch.equal(resulta.b[1].cpu()			,					_lowercase))
								self.assertEqual(resulta.c			,					1)

								A_       		=   send_to_device(UserDict({'a': tensor, 'b': [tensor, tensor], 'c': 1})			,					_lowercase)
								self.assertIsInstance(_lowercase			,					_lowercase)
								self.assertTrue(torch.equal(resulta['a'].cpu()			,					_lowercase))
								self.assertIsInstance(resulta['b']			,					_lowercase)
								self.assertTrue(torch.equal(resulta['b'][0].cpu()			,					_lowercase))
								self.assertTrue(torch.equal(resulta['b'][1].cpu()			,					_lowercase))
								self.assertEqual(resulta['c']			,					1)



					def 						__snake_case	(  self    :	Dict)						->       str:
								with self.assertRaises(_lowercase) as cm:
											A_       		=   recursively_apply(torch.tensor			,					(torch.tensor(1), 1)			,					error_on_other_type=_lowercase)
								self.assertEqual(
								    str(cm.exception)			,					'Unsupported types (<class \'int\'>) passed to `tensor`. Only nested list/tuple/dicts of objects that are valid for `is_torch_tensor` should be passed.'			,					)



					def 						__snake_case	(  self    :	Union[str, Any])						->       List[str]:
								A_       		=   torch.tensor([1, 2, 3, 4, 5])
								self.assertEqual(listify(_lowercase)			,					[1, 2, 3, 4, 5])

								A_       		=   torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]])
								self.assertEqual(listify(_lowercase)			,					[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]])

								A_       		=   torch.tensor([[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]], [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20]]])
								self.assertEqual(
								    listify(_lowercase)			,					[[[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]], [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20]]])



					def 						__snake_case	(  self    :	List[Any])						->       Union[str, Any]:
								with patch_environment(aa=1			,					BB=2):
											self.assertEqual(os.environ.get('AA')			,					'1')
											self.assertEqual(os.environ.get('BB')			,					'2')

								self.assertNotIn('AA'			,					os.environ)
								self.assertNotIn('BB'			,					os.environ)



					def 						__snake_case	(  self    :	Any)						->       str:
								A_       		=   RegressionModel()
								A_       		=   model.forward
								A_       		=   convert_outputs_to_fpaa(model.forward)
								A_       		=   extract_model_from_parallel(_lowercase			,					keep_fpaa_wrapper=_lowercase)
								A_       		=   pickle.dumps(_lowercase)



					@require_cuda
					def 						__snake_case	(  self    :	Union[str, Any])						->       Union[str, Any]:
								A_       		=   RegressionModel()
								A_       		=   model.forward
								A_       		=   torch.cuda.amp.autocast(dtype=torch.floataa)(model.forward)
								A_       		=   convert_outputs_to_fpaa(model.forward)
								A_       		=   extract_model_from_parallel(_lowercase			,					keep_fpaa_wrapper=_lowercase)
								A_       		=   pickle.dumps(_lowercase)



					@require_cuda
					@require_torch_min_version(version='2.0')
					def 						__snake_case	(  self    :	Optional[Any])						->       Dict:
								A_       		=   RegressionModel()
								A_       		=   model.forward
								A_       		=   torch.cuda.amp.autocast(dtype=torch.floataa)(model.forward)
								A_       		=   convert_outputs_to_fpaa(model.forward)
								A_       		=   torch.compile(model.forward			,					backend='inductor')
								A_       		=   torch.randn(4			,					10).cuda()
								A_       		=   model(_lowercase)



					def 						__snake_case	(  self    :	Union[str, Any])						->       Optional[Any]:
								A_       		=   RegressionModel()
								# could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
								A_       		=   torch.nn.parallel.DataParallel(_lowercase)
								A_       		=   extract_model_from_parallel(_lowercase)

								self.assertEqual(_lowercase			,					_lowercase)



					@require_torch_min_version(version='2.0')
					def 						__snake_case	(  self    :	str)						->       int:
								A_       		=   RegressionModel()
								A_       		=   torch.compile(_lowercase)

								# could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
								A_       		=   torch.nn.parallel.DataParallel(_lowercase)
								A_       		=   torch.compile(_lowercase)
								A_       		=   extract_model_from_parallel(_lowercase)

								self.assertEqual(compiled_model._orig_mod			,					compiled_model_unwrapped._orig_mod)



					def 						__snake_case	(  self    :	List[str])						->       Optional[int]:
								self.assertEqual(find_device([1, 'a', torch.tensor([1, 2, 3])])			,					torch.device('cpu'))
								self.assertEqual(find_device({'a': 1, 'b': torch.tensor([1, 2, 3])})			,					torch.device('cpu'))
								self.assertIsNone(find_device([1, 'a']))




