




"""simple docstring"""







import argparse
import pathlib
from pathlib import Path
from tempfile import TemporaryDirectory

import esm as esm_module
import torch
from esm.esmfold.va.misc import batch_encode_sequences as esmfold_encode_sequences
from esm.esmfold.va.pretrained import esmfold_va

from transformers.models.esm.configuration_esm import EsmConfig, EsmFoldConfig
from transformers.models.esm.modeling_esm import (
    EsmForMaskedLM,
    EsmForSequenceClassification,
    EsmIntermediate,
    EsmLayer,
    EsmOutput,
    EsmSelfAttention,
    EsmSelfOutput,
)
from transformers.models.esm.modeling_esmfold import EsmForProteinFolding
from transformers.models.esm.tokenization_esm import EsmTokenizer
from transformers.utils import logging


logging.set_verbosity_info()
SCREAMING_SNAKE_CASE_      =					logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_      =					[
    (
        'protein1',
        'MNGTEGPNFYVPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSRYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQFRNCMLTTICCGKNPLGDDEASATVSKTETSQVAPA',
    ),
    ('protein2', 'MKTVRQERLKSIVRILERSKEPVSGAQLAEELSVSRQVIVQDIAYLRSLGYNIVATPRGYVLA'),
    ('protein3', 'MKTVRQERLKSI<mask>RILERSKEPVSGAQLAEELS<mask>SRQVIVQDIAYLRSLGYN<mask>VATPRGYVLAGG'),
    ('protein4', 'MKTVRQERLKSI<mask>RILERSKEPVSGAQLAEELS<mask>SRQVIVQDIAYLRSLGYN<mask>VATPRGYVLA'),
]

SCREAMING_SNAKE_CASE_      =					{
    'esm1b_t33_650M_UR50S': esm_module.pretrained.esm1b_t33_650M_UR50S,
    'esm1v_t33_650M_UR90S_1': esm_module.pretrained.esm1v_t33_650M_UR90S_1,
    'esm1v_t33_650M_UR90S_2': esm_module.pretrained.esm1v_t33_650M_UR90S_2,
    'esm1v_t33_650M_UR90S_3': esm_module.pretrained.esm1v_t33_650M_UR90S_3,
    'esm1v_t33_650M_UR90S_4': esm_module.pretrained.esm1v_t33_650M_UR90S_4,
    'esm1v_t33_650M_UR90S_5': esm_module.pretrained.esm1v_t33_650M_UR90S_5,
    'esm2_t48_15B_UR50D': esm_module.pretrained.esm2_t48_15B_UR50D,
    'esm2_t36_3B_UR50D': esm_module.pretrained.esm2_t36_3B_UR50D,
    'esm2_t33_650M_UR50D': esm_module.pretrained.esm2_t33_650M_UR50D,
    'esm2_t30_150M_UR50D': esm_module.pretrained.esm2_t30_150M_UR50D,
    'esm2_t12_35M_UR50D': esm_module.pretrained.esm2_t12_35M_UR50D,
    'esm2_t6_8M_UR50D': esm_module.pretrained.esm2_t6_8M_UR50D,
    'esmfold_v1': esmfold_va,
}

SCREAMING_SNAKE_CASE_      =					list('ARNDCQEGHILKMFPSTWYV')

SCREAMING_SNAKE_CASE_      =					restypes + ['X']
SCREAMING_SNAKE_CASE_      =					restypes_with_x + ['<pad>', '<mask>', '<cls>', '<sep>', '<eos>']





def     __snake_case   (							):


    """simple docstring"""







    with TemporaryDirectory() as tempdir:
        UpperCamelCase										=  '''\n'''.join(_lowercase						)
        UpperCamelCase										=  Path(_lowercase						) / '''vocab.txt'''
        vocab_file.write_text(_lowercase						)
        UpperCamelCase										=  EsmTokenizer(vocab_file=str(_lowercase						)						)
    UpperCamelCase										=  0  # Overlaps with 'A' but that seems to be what they want
    return hf_tokenizer





def     __snake_case   (							_lowercase ,_lowercase						):


    """simple docstring"""







    UpperCamelCase										=  our_module.load_state_dict(original_module.state_dict()						)
    if status.missing_keys:
        raise ValueError(f'Missing keys: {status.missing_keys}'						)
    if status.unexpected_keys:
        raise ValueError(f'Unexpected keys: {status.unexpected_keys}'						)





def     __snake_case   (							_lowercase ,_lowercase ,_lowercase ,_lowercase ,_lowercase						):


    """simple docstring"""







    if model.startswith('''esmfold'''						):
        UpperCamelCase										=  MODEL_MAPPING[model]()
    else:
        UpperCamelCase					,  UpperCamelCase										=  MODEL_MAPPING[model]()
    esm.eval()  # disable dropout

    if model.startswith('''esmfold'''						):
        UpperCamelCase										=  esm.esm.embed_dim
        UpperCamelCase										=  esm.esm.num_layers
        UpperCamelCase										=  esm.esm.attention_heads
        UpperCamelCase										=  4 * embed_dim
        UpperCamelCase										=  esm.esm.token_dropout
        UpperCamelCase										=  False  # This code path does not exist in ESM-2
        UpperCamelCase										=  '''rotary'''
        UpperCamelCase										=  True
        UpperCamelCase										=  EsmFoldConfig()
        for key, val in esm.cfg.items():
            if hasattr(_lowercase ,_lowercase						) and key != "trunk":
                setattr(_lowercase ,_lowercase ,_lowercase						)
        for key, val in esm.cfg.trunk.items():
            if hasattr(esmfold_config.trunk ,_lowercase						) and key != "structure_module":
                setattr(esmfold_config.trunk ,_lowercase ,_lowercase						)
        for key, val in esm.cfg.trunk.structure_module.items():
            if hasattr(esmfold_config.trunk.structure_module ,_lowercase						):
                setattr(esmfold_config.trunk.structure_module ,_lowercase ,_lowercase						)
    elif hasattr(_lowercase ,'''args'''						):
        # Indicates an ESM-1b or ESM-1v model
        UpperCamelCase										=  esm.args.embed_dim
        UpperCamelCase										=  esm.args.layers
        UpperCamelCase										=  esm.args.attention_heads
        UpperCamelCase										=  esm.args.ffn_embed_dim
        UpperCamelCase										=  esm.args.token_dropout
        UpperCamelCase										=  True if esm.emb_layer_norm_before else False
        UpperCamelCase										=  '''absolute'''
        UpperCamelCase										=  False
        UpperCamelCase										=  None
    else:
        # Indicates an ESM-2 model
        UpperCamelCase										=  esm.embed_dim
        UpperCamelCase										=  esm.num_layers
        UpperCamelCase										=  esm.attention_heads
        UpperCamelCase										=  4 * embed_dim  # This is hardcoded in ESM-2
        UpperCamelCase										=  esm.token_dropout
        UpperCamelCase										=  False  # This code path does not exist in ESM-2
        UpperCamelCase										=  '''rotary'''
        UpperCamelCase										=  False
        UpperCamelCase										=  None

    if is_folding_model:
        UpperCamelCase										=  esm.esm.alphabet
    UpperCamelCase										=  tuple(alphabet.all_toks						)
    UpperCamelCase										=  alphabet.mask_idx
    UpperCamelCase										=  alphabet.padding_idx

    if is_folding_model:
        UpperCamelCase										=  esm.esm
    else:
        UpperCamelCase										=  esm

    UpperCamelCase										=  EsmConfig(
        vocab_size=original_esm_model.embed_tokens.num_embeddings ,mask_token_id=_lowercase ,hidden_size=_lowercase ,num_hidden_layers=_lowercase ,num_attention_heads=_lowercase ,intermediate_size=_lowercase ,max_position_embeddings=1026 ,layer_norm_eps=1e-5 ,attention_probs_dropout_prob=0.0 ,hidden_dropout_prob=0.0 ,pad_token_id=_lowercase ,emb_layer_norm_before=_lowercase ,token_dropout=_lowercase ,position_embedding_type=_lowercase ,is_folding_model=_lowercase ,esmfold_config=_lowercase ,vocab_list=_lowercase ,)
    if classification_head:
        UpperCamelCase										=  esm.classification_heads['''mnli'''].out_proj.weight.shape[0]
    print('''Our ESM config:''' ,_lowercase						)

    if model.startswith('''esmfold'''						):
        UpperCamelCase										=  EsmForProteinFolding
    elif classification_head:
        UpperCamelCase										=  EsmForSequenceClassification
    else:
        UpperCamelCase										=  EsmForMaskedLM
    UpperCamelCase										=  model_class(_lowercase						)
    model.eval()

    # Now let's copy all the weights.
    # Embeddings
    UpperCamelCase										=  original_esm_model.embed_tokens.weight
    if position_embedding_type == "absolute":
        UpperCamelCase										=  original_esm_model.embed_positions.weight

    if config.emb_layer_norm_before:
        UpperCamelCase										=  original_esm_model.emb_layer_norm_before.weight
        UpperCamelCase										=  original_esm_model.emb_layer_norm_before.bias

    UpperCamelCase										=  original_esm_model.emb_layer_norm_after.weight
    UpperCamelCase										=  original_esm_model.emb_layer_norm_after.bias

    for i in range(config.num_hidden_layers						):
        # Encoder: start of layer
        UpperCamelCase										=  model.esm.encoder.layer[i]
        # esm_layer: TransformerSentenceEncoderLayer = original_esm_model.layers[i]
        UpperCamelCase										=  original_esm_model.layers[i]

        # self attention
        UpperCamelCase										=  layer.attention.self
        assert (
            esm_layer.self_attn.k_proj.weight.data.shape
            == esm_layer.self_attn.q_proj.weight.data.shape
            == esm_layer.self_attn.v_proj.weight.data.shape
            == torch.Size((config.hidden_size, config.hidden_size)						)
        )

        UpperCamelCase										=  esm_layer.self_attn.q_proj.weight
        UpperCamelCase										=  esm_layer.self_attn.q_proj.bias
        UpperCamelCase										=  esm_layer.self_attn.k_proj.weight
        UpperCamelCase										=  esm_layer.self_attn.k_proj.bias
        UpperCamelCase										=  esm_layer.self_attn.v_proj.weight
        UpperCamelCase										=  esm_layer.self_attn.v_proj.bias

        if getattr(esm_layer.self_attn ,'''rot_emb''' ,_lowercase						) is not None:
            # Matt: Although inv_freq is not a trainable weight, it is computed at model init and cached.
            # During the training of ESM-2 the model was converted to float16 precision, which also converts
            # the inv_freq tensor, and the loss of precision remains even if the model is loaded later as float32.
            # If we recompute inv_freq without this loss of precision then we will get subtly different rotary
            # embeddings, which are enough to cause significant discrepancies in model outputs. To avoid this,
            # we make sure the new model copies the data from the old inv_freq.
            UpperCamelCase										=  esm_layer.self_attn.rot_emb.inv_freq

        # LayerNorm changes for pre-activation
        UpperCamelCase										=  esm_layer.self_attn_layer_norm.weight
        UpperCamelCase										=  esm_layer.self_attn_layer_norm.bias
        UpperCamelCase										=  esm_layer.final_layer_norm.weight
        UpperCamelCase										=  esm_layer.final_layer_norm.bias

        # self-attention output
        UpperCamelCase										=  layer.attention.output
        assert self_output.dense.weight.shape == esm_layer.self_attn.out_proj.weight.shape
        UpperCamelCase										=  esm_layer.self_attn.out_proj.weight
        UpperCamelCase										=  esm_layer.self_attn.out_proj.bias

        # intermediate
        UpperCamelCase										=  layer.intermediate
        assert intermediate.dense.weight.shape == esm_layer.fca.weight.shape
        UpperCamelCase										=  esm_layer.fca.weight
        UpperCamelCase										=  esm_layer.fca.bias

        # output
        UpperCamelCase										=  layer.output
        assert bert_output.dense.weight.shape == esm_layer.fca.weight.shape
        UpperCamelCase										=  esm_layer.fca.weight
        UpperCamelCase										=  esm_layer.fca.bias
        # end of layer

    if is_folding_model:
        UpperCamelCase										=  esm.esm_s_combine.data
        UpperCamelCase										=  esm.afa_to_esm.data
        transfer_and_check_weights(esm.embedding ,model.embedding						)
        transfer_and_check_weights(esm.esm_s_mlp ,model.esm_s_mlp						)
        transfer_and_check_weights(esm.trunk ,model.trunk						)
        transfer_and_check_weights(esm.distogram_head ,model.distogram_head						)
        transfer_and_check_weights(esm.ptm_head ,model.ptm_head						)
        transfer_and_check_weights(esm.lm_head ,model.lm_head						)
        transfer_and_check_weights(esm.lddt_head ,model.lddt_head						)

    elif classification_head:
        UpperCamelCase										=  esm.esm.classification_heads['''mnli'''].dense.weight
        UpperCamelCase										=  esm.classification_heads['''mnli'''].dense.bias
        UpperCamelCase										=  esm.classification_heads['''mnli'''].out_proj.weight
        UpperCamelCase										=  esm.classification_heads['''mnli'''].out_proj.bias
    else:
        # LM Head
        UpperCamelCase										=  esm.lm_head.dense.weight
        UpperCamelCase										=  esm.lm_head.dense.bias
        UpperCamelCase										=  esm.lm_head.layer_norm.weight
        UpperCamelCase										=  esm.lm_head.layer_norm.bias
        UpperCamelCase										=  esm.lm_head.weight
        UpperCamelCase										=  esm.lm_head.bias

    # Contact prediction head
    transfer_and_check_weights(esm.contact_head ,model.esm.contact_head						)

    # Prepare data (first 2 sequences from ESMStructuralSplitDataset superfamily / 4)
    if is_folding_model:
        # Folding models aren't trained on masked inputs and don't like mask tokens.
        UpperCamelCase										=  SAMPLE_DATA[:2]
    else:
        UpperCamelCase										=  SAMPLE_DATA

    if is_folding_model:
        UpperCamelCase										=  get_esmfold_tokenizer()
        UpperCamelCase										=  hf_tokenizer(
            [row[1] for row in sample_data] ,return_tensors='''pt''' ,padding=_lowercase ,add_special_tokens=_lowercase						)
        UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase										=  esmfold_encode_sequences([row[1] for row in sample_data]						)
        UpperCamelCase										=  torch.all(hf_tokens['''input_ids'''] == esmfold_aas						) and torch.all(
            hf_tokens['''attention_mask'''] == esmfold_mask						)
    else:
        # Let's check that we get the same results.
        UpperCamelCase										=  alphabet.get_batch_converter()
        UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase										=  batch_converter(_lowercase						)
        # Prepare tokenizer and make sure it matches
        with TemporaryDirectory() as tempdir:
            UpperCamelCase										=  '''\n'''.join(alphabet.all_toks						)
            UpperCamelCase										=  Path(_lowercase						) / '''vocab.txt'''
            vocab_file.write_text(_lowercase						)
            UpperCamelCase										=  EsmTokenizer(vocab_file=str(_lowercase						)						)

        UpperCamelCase										=  hf_tokenizer([row[1] for row in sample_data] ,return_tensors='''pt''' ,padding=_lowercase						)
        UpperCamelCase										=  torch.all(hf_tokens['''input_ids'''] == batch_tokens						)

    print('''Do both models tokenizers output the same tokens?''' ,'''🔥''' if success else '''💩'''						)
    if not success:
        raise Exception('''Tokenization does not match!'''						)

    with torch.no_grad():
        if is_folding_model:
            # Let's test the model in parts
            # ESMFold always converts the ESM stem to float16, which requires float16 ops
            # that don't exist on CPU. Therefore, to test it we need to run it on GPU. However,
            # ESMFold is what we in the community call a "big boy" and so we desperately avoid putting both the
            # original and the converted model on the GPU at the same time.
            UpperCamelCase										=  esm.cuda().infer([row[1] for row in sample_data]						)
            UpperCamelCase										=  model.cuda()(
                input_ids=hf_tokens['''input_ids'''].cuda() ,attention_mask=hf_tokens['''attention_mask'''].cuda()						)
        else:
            UpperCamelCase										=  model(**_lowercase ,output_hidden_states=_lowercase						)
            UpperCamelCase										=  our_output['''logits''']
            if classification_head:
                UpperCamelCase										=  esm.model.classification_heads['''mnli'''](esm.extract_features(_lowercase						)						)
            else:
                UpperCamelCase										=  esm(hf_tokens['''input_ids'''] ,repr_layers=list(range(999						)						)						)
                UpperCamelCase										=  their_output['''logits''']

        if is_folding_model:
            UpperCamelCase										=  torch.max(torch.abs(our_output['''positions'''] - their_output['''positions''']						)						).item()
            UpperCamelCase										=  torch.allclose(our_output['''positions'''] ,their_output['''positions'''] ,atol=1e-5						)
        else:
            UpperCamelCase										=  torch.max(torch.abs(our_output - their_output						)						).item()
            UpperCamelCase										=  torch.allclose(_lowercase ,_lowercase ,atol=1e-5						)

        print(f'max_absolute_diff = {max_absolute_diff}'						)  # ~ 1e-5
        print('''Do both models output the same tensors?''' ,'''🔥''' if success else '''💩'''						)

        if not success:
            raise Exception('''Something went wRoNg'''						)

        if not is_folding_model:
            # Let's check contact prediction too
            UpperCamelCase										=  model.predict_contacts(hf_tokens['''input_ids'''] ,hf_tokens['''attention_mask''']						)
            UpperCamelCase										=  esm.predict_contacts(hf_tokens['''input_ids''']						)
            UpperCamelCase										=  torch.max(torch.abs(our_output - their_output						)						).item()
            UpperCamelCase										=  torch.allclose(_lowercase ,_lowercase ,atol=1e-5						)

            print('''Contact prediction testing:'''						)
            print(f'max_absolute_diff = {max_absolute_diff}'						)  # ~ 1e-5
            print('''Do both models output the same tensors?''' ,'''🔥''' if success else '''💩'''						)

            if not success:
                raise Exception('''Something went wRoNg'''						)

        pathlib.Path(_lowercase						).mkdir(parents=_lowercase ,exist_ok=_lowercase						)
        print(f'Saving model to {pytorch_dump_folder_path}'						)
        model.save_pretrained(_lowercase						)

        del esm  # Free up some memory before continuing

    print(f'Saving tokenizer to {pytorch_dump_folder_path}'						)
    hf_tokenizer.save_pretrained(_lowercase						)

    if push_to_repo:
        model.push_to_hub(repo_id=_lowercase ,use_auth_token=_lowercase						)
        hf_tokenizer.push_to_hub(repo_id=_lowercase ,use_auth_token=_lowercase						)


if __name__ == "__main__":
  SCREAMING_SNAKE_CASE_      =					argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      '--pytorch_dump_folder_path', type=str, required=True, help='Path to the output PyTorch model.'
  )
  parser.add_argument(
      '--classification_head', action='store_true', help='Whether to convert a final classification head.'
  )
  parser.add_argument('--model', default=None, type=str, required=True, help='Name of model to convert.')
  parser.add_argument('--push_to_repo', type=str, help='Repo to upload to (including username!).')
  parser.add_argument('--auth_token', type=str, help='HuggingFace auth token.')
  SCREAMING_SNAKE_CASE_      =					parser.parse_args()
  convert_esm_checkpoint_to_pytorch(
      args.model, args.pytorch_dump_folder_path, args.classification_head, args.push_to_repo, args.auth_token
  )





"""simple docstring"""







import torch

from transformers import PreTrainedModel

from .custom_configuration import CustomConfig, NoSuperInitConfig

class 							snake_case_ (				lowerCamelCase_ ):







   """simple docstring"""




   A_										=					CustomConfig







   def __init__(		self					,				lowerCamelCase_)							->      List[Any]:
       super().__init__(lowerCamelCase_)
       UpperCamelCase										=  torch.nn.Linear(config.hidden_size					,				config.hidden_size)







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_)							->      Optional[Any]:
       return self.linear(lowerCamelCase_)







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_)							->      List[str]:
       pass

class 							snake_case_ (				lowerCamelCase_ ):







   """simple docstring"""




   A_										=					NoSuperInitConfig







   def __init__(		self					,				lowerCamelCase_)							->      Optional[int]:
       super().__init__(lowerCamelCase_)
       UpperCamelCase										=  torch.nn.Linear(config.attribute					,				config.attribute)







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_)							->      int:
       return self.linear(lowerCamelCase_)







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_)							->      Any:
       pass