
# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import List


def 					__lowerCAmelCase    (    a__		,					a__						) ->						Dict:
				if name != "training_function" and name != "main":
								raise ValueError(F"""Incorrect function name passed: {name}, choose either 'main' or 'training_function'"""						)
				__a							,  __a    							=					[], False
				for line in lines:
								if not found_start and F"""def {name}""" in line:
												__a    							=					True
												good_lines.append(a__						)
												continue
								if found_start:
												if name == "training_function" and "def main" in line:
																return good_lines
												if name == "main" and "if __name__" in line:
																return good_lines
												good_lines.append(a__						)


def 					__lowerCAmelCase    (    a__						) ->						Union[str, Any]:
				return [line for line in lines if not line.lstrip().startswith('''#'''						) and line != "\n"]






def 					__lowerCAmelCase    (    a__		,					a__		,					a__		,					a__ = None						) ->						Optional[int]:
				with open(a__		,					'''r'''						) as f:
								__a    							=					f.readlines()
				with open(os.path.abspath(os.path.join('''examples'''		,					'''nlp_example.py'''						)						)		,					'''r'''						) as f:
								__a    							=					f.readlines()
				with open(a__		,					'''r'''						) as f:
								__a    							=					f.readlines()
				if secondary_filename is not None:
								with open(a__		,					'''r'''						) as f:
												__a    							=					f.readlines()

    # This is our base, we remove all the code from here in our `full_filename` and `feature_filename` to find the new content
				if parser_only:
								__a    							=					clean_lines(get_function_contents_by_name(a__		,					'''main'''						)						)
								__a    							=					clean_lines(get_function_contents_by_name(a__		,					'''main'''						)						)
								__a    							=					clean_lines(get_function_contents_by_name(a__		,					'''main'''						)						)
								if secondary_filename is not None:
												__a    							=					clean_lines(get_function_contents_by_name(a__		,					'''main'''						)						)
				else:
								__a    							=					clean_lines(get_function_contents_by_name(a__		,					'''training_function'''						)						)
								__a    							=					clean_lines(get_function_contents_by_name(a__		,					'''training_function'''						)						)
								__a    							=					clean_lines(get_function_contents_by_name(a__		,					'''training_function'''						)						)
								if secondary_filename is not None:
												__a    							=					clean_lines(
												    get_function_contents_by_name(a__		,					'''training_function'''						)						)

				__a    							=					'''train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)\n'''

				# Specific code in our script that differs from the full version, aka what is new
				__a    							=					[]
				__a    							=					[]  # We keep track of the idxs just in case it's a repeated statement
				__a    							=					iter(a__						)
				for i in range(len(a__						) - 1						):
								if i not in passed_idxs:
												__a    							=					next(a__						)
												if (line not in full_file_func) and (line.lstrip() != _dl_line):
																if "TESTING_MOCKED_DATALOADERS" not in line:
																				new_feature_code.append(a__						)
																				passed_idxs.append(a__						)
																else:
																				# Skip over the `config['num_epochs'] = 2` statement
																				__a    							=					next(a__						)

    # Extract out just the new parts from the full_file_training_func
				__a    							=					[]
				__a    							=					[]  # We keep track of the idxs just in case it's a repeated statement
				for i, line in enumerate(a__						):
								if i not in passed_idxs:
												if (line not in full_file_func) and (line.lstrip() != _dl_line):
																if "TESTING_MOCKED_DATALOADERS" not in line:
																				new_full_example_parts.append(a__						)
																				passed_idxs.append(a__						)

    # Finally, get the overall diff
				__a    							=					[line for line in new_feature_code if line not in new_full_example_parts]
				if secondary_filename is not None:
								__a    							=					[line for line in full_file_contents if line not in secondary_file_func]
								__a    							=					[line for line in diff_from_example if line not in diff_from_two]

				return diff_from_example

import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


A  :			List[str]     				=				logging.get_logger(__name__)







class     __A(				nn.Module    ):




				def __init__(  self       )					->   Union[str, Any]:


								'''simple docstring'''




								super().__init__()
								if version.parse(torch.__version__       ) < version.parse('''1.12.0'''       ):
												raise ImportError(
												    F"""You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use """
												    '''PytorchGELUTanh. Please upgrade torch.'''       )




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return nn.functional.gelu(_snake_case	,  approximate='''tanh'''       )







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi       ) * (input + 0.04_4715 * torch.pow(_snake_case	,  3.0       ))       ))







class     __A(				nn.Module    ):




				def __init__(  self	,  _snake_case = False       )					->   Tuple:


								'''simple docstring'''




								super().__init__()
								if use_gelu_python:
												__a    							=					self._gelu_python
								else:
												__a    							=					nn.functional.gelu




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0       )       ))




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return self.act(_snake_case       )







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return 0.5 * input * (1.0 + torch.tanh(input * 0.79_7884_5608 * (1.0 + 0.04_4715 * input * input)       ))







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return input * torch.sigmoid(1.702 * input       )







class     __A(				nn.Module    ):




				def __init__(  self	,  _snake_case	,  _snake_case       )					->   Optional[int]:


								'''simple docstring'''




								if min > max:
												raise ValueError(F"""min should be < max (got min: {min}, max: {max})"""       )

								super().__init__()
								__a    							=					min
								__a    							=					max




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return torch.clip(gelu(_snake_case       )	,  self.min	,  self.max       )







class     __A(				nn.Module    ):




				def __init__(  self       )					->   str:


								'''simple docstring'''




								super().__init__()
								__a    							=					math.sqrt(2 / math.pi       )




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_4715 * torch.pow(_snake_case	,  3       ))       ))







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return nn.functional.silu(_snake_case       )







class     __A(				nn.Module    ):




				def __init__(  self       )					->   List[Any]:


								'''simple docstring'''




								super().__init__()
								if version.parse(torch.__version__       ) < version.parse('''1.9.0'''       ):
												__a    							=					self._mish_python
								else:
												__a    							=					nn.functional.mish




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return input * torch.tanh(nn.functional.softplus(_snake_case       )       )




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return self.act(_snake_case       )







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Tensor:


								'''simple docstring'''




								return input







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case	,  _snake_case=0.70_7107	,  _snake_case=0.28_2095       )					->   int:


								'''simple docstring'''




								__a    							=					(input - mu).div(sigma * math.sqrt(2.0       )       )
								return 0.5 * (1.0 + torch.erf(_snake_case       ))







class     __A(				nn.Module    ):




				def        SCREAMING_SNAKE_CASE_	(  self	,  _snake_case       )					->   Any:


								'''simple docstring'''




								__a    							=					nn.functional.relu(_snake_case       )
								__a    							=					torch.square(_snake_case       )
								return squared







class     __A(				a    ):




				def __getitem__(  self	,  _snake_case       )					->   Dict:


								'''simple docstring'''




								__a    							=					super().__getitem__(_snake_case       )
								__a							,  __a    							=					content if isinstance(_snake_case	,  _snake_case       ) else (content, {})
								return cls(**_snake_case       )



A  :			str     				=				{
    'gelu': GELUActivation,
    'gelu_10': (ClippedGELUActivation, {'min': -1_0, 'max': 1_0}),
    'gelu_fast': FastGELUActivation,
    'gelu_new': NewGELUActivation,
    'gelu_python': (GELUActivation, {'use_gelu_python': True}),
    'gelu_pytorch_tanh': PytorchGELUTanh,
    'gelu_accurate': AccurateGELUActivation,
    'laplace': LaplaceActivation,
    'linear': LinearActivation,
    'mish': MishActivation,
    'quick_gelu': QuickGELUActivation,
    'relu': nn.ReLU,
    'relu2': ReLUSquaredActivation,
    'relu6': nn.ReLUa,
    'sigmoid': nn.Sigmoid,
    'silu': SiLUActivation,
    'swish': SiLUActivation,
    'tanh': nn.Tanh,
}
A  :			Any     				=				ClassInstantier(ACTaCLS)


def 					__lowerCAmelCase    (    a__						) ->						Optional[int]:
				if activation_string in ACTaFN:
								return ACTaFN[activation_string]
				else:
								raise KeyError(F"""function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()						)}"""						)


# For backwards compatibility with: from activations import gelu_python
A  :			Optional[Any]     				=				get_activation('gelu_python')
A  :			str     				=				get_activation('gelu_new')
A  :			Any     				=				get_activation('gelu')
A  :			int     				=				get_activation('gelu_fast')
A  :			Any     				=				get_activation('quick_gelu')
A  :			int     				=				get_activation('silu')
A  :			List[str]     				=				get_activation('mish')






A  :			Dict     				=				get_activation('linear')