




'''simple docstring'''




import argparse
import logging
import math
import time

import torch

from transformers import TransfoXLCorpus, TransfoXLLMHeadModel


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
UpperCAmelCase_       				=      logging.getLogger(__name__)

def    _UpperCamelCase      (				):
 '''simple docstring'''





 UpperCAmelCase__										=    argparse.ArgumentParser(description="""PyTorch Transformer Language Model"""       )
 parser.add_argument("""--model_name"""       ,						type=SCREAMING_SNAKE_CASE__       ,						default="""transfo-xl-wt103"""       ,						help="""pretrained model name"""       )
 parser.add_argument(
     """--split"""       ,						type=SCREAMING_SNAKE_CASE__       ,						default="""test"""       ,						choices=["""all""", """valid""", """test"""]       ,						help="""which split to evaluate"""       )
 parser.add_argument("""--batch_size"""       ,						type=SCREAMING_SNAKE_CASE__       ,						default=10       ,						help="""batch size"""       )
 parser.add_argument("""--tgt_len"""       ,						type=SCREAMING_SNAKE_CASE__       ,						default=128       ,						help="""number of tokens to predict"""       )
 parser.add_argument("""--ext_len"""       ,						type=SCREAMING_SNAKE_CASE__       ,						default=0       ,						help="""length of the extended context"""       )
 parser.add_argument("""--mem_len"""       ,						type=SCREAMING_SNAKE_CASE__       ,						default=1600       ,						help="""length of the retained previous heads"""       )
 parser.add_argument("""--clamp_len"""       ,						type=SCREAMING_SNAKE_CASE__       ,						default=1000       ,						help="""max positional embedding index"""       )
 parser.add_argument("""--no_cuda"""       ,						action="""store_true"""       ,						help="""Do not use CUDA even though CUA is available"""       )
 parser.add_argument("""--work_dir"""       ,						type=SCREAMING_SNAKE_CASE__       ,						required=SCREAMING_SNAKE_CASE__       ,						help="""path to the work_dir"""       )
 parser.add_argument("""--no_log"""       ,						action="""store_true"""       ,						help="""do not log the eval result"""       )
 parser.add_argument("""--same_length"""       ,						action="""store_true"""       ,						help="""set same length attention with masking"""       )
 parser.add_argument("""--server_ip"""       ,						type=SCREAMING_SNAKE_CASE__       ,						default=""""""       ,						help="""Can be used for distant debugging."""       )
 parser.add_argument("""--server_port"""       ,						type=SCREAMING_SNAKE_CASE__       ,						default=""""""       ,						help="""Can be used for distant debugging."""       )
 UpperCAmelCase__										=    parser.parse_args()
 assert args.ext_len >= 0, "extended context length must be non-negative"

 if args.server_ip and args.server_port:
  # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
  import ptvsd

  print("""Waiting for debugger attach"""       )
  ptvsd.enable_attach(address=(args.server_ip, args.server_port)       ,						redirect_output=SCREAMING_SNAKE_CASE__       )
  ptvsd.wait_for_attach()

 UpperCAmelCase__										=    torch.device("""cuda""" if torch.cuda.is_available() and not args.no_cuda else """cpu"""       )
 logger.info("""device: {}""".format(SCREAMING_SNAKE_CASE__       )       )

 # Load a pre-processed dataset
 # You can also build the corpus yourself using TransfoXLCorpus methods
 # The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
 # and tokenizing the dataset
 # The pre-processed corpus is a convertion (using the conversion script )
 UpperCAmelCase__										=    TransfoXLCorpus.from_pretrained(args.model_name       )

 UpperCAmelCase__										=    corpus.get_iterator("""valid"""       ,						args.batch_size       ,						args.tgt_len       ,						device=SCREAMING_SNAKE_CASE__       ,						ext_len=args.ext_len       )
 UpperCAmelCase__										=    corpus.get_iterator("""test"""       ,						args.batch_size       ,						args.tgt_len       ,						device=SCREAMING_SNAKE_CASE__       ,						ext_len=args.ext_len       )

 # Load a pre-trained model
 UpperCAmelCase__										=    TransfoXLLMHeadModel.from_pretrained(args.model_name       )
 model.to(SCREAMING_SNAKE_CASE__       )

 logger.info(
     """Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}""".format(
         args.batch_size       ,						args.tgt_len       ,						args.ext_len       ,						args.mem_len       ,						args.clamp_len       )       )

 model.reset_memory_length(args.mem_len       )
 if args.clamp_len > 0:
  UpperCAmelCase__										=    args.clamp_len
 if args.same_length:
  UpperCAmelCase__										=    True

 ###############################################################################
 # Evaluation code
 ###############################################################################
 def evaluate(SCREAMING_SNAKE_CASE__			:      Union[str, Any]       ):
  # Turn on evaluation mode which disables dropout.
  model.eval()
  UpperCAmelCase__							,			UpperCAmelCase__										=    0, 0.0
  UpperCAmelCase__										=    time.time()
  with torch.no_grad():
   UpperCAmelCase__										=    None
   for idx, (data, target, seq_len) in enumerate(SCREAMING_SNAKE_CASE__       ):
    UpperCAmelCase__										=    model(SCREAMING_SNAKE_CASE__       ,						lm_labels=SCREAMING_SNAKE_CASE__       ,						mems=SCREAMING_SNAKE_CASE__       )
    UpperCAmelCase__							,			UpperCAmelCase__							,			UpperCAmelCase__										=    ret
    UpperCAmelCase__										=    loss.mean()
    total_loss += seq_len * loss.item()
    total_len += seq_len
   UpperCAmelCase__										=    time.time() - start_time
  logger.info("""Time : {:.2f}s, {:.2f}ms/segment""".format(SCREAMING_SNAKE_CASE__       ,						1000 * total_time / (idx + 1)       )       )
  return total_loss / total_len

 # Run on test data.
 if args.split == "all":
  UpperCAmelCase__										=    evaluate(SCREAMING_SNAKE_CASE__       )
  UpperCAmelCase__										=    evaluate(SCREAMING_SNAKE_CASE__       )
 elif args.split == "valid":
  UpperCAmelCase__										=    evaluate(SCREAMING_SNAKE_CASE__       )
  UpperCAmelCase__										=    None
 elif args.split == "test":
  UpperCAmelCase__										=    evaluate(SCREAMING_SNAKE_CASE__       )
  UpperCAmelCase__										=    None

 def format_log(SCREAMING_SNAKE_CASE__			:      List[Any]       ,						SCREAMING_SNAKE_CASE__			:      List[str]       ):
  UpperCAmelCase__										=    """| {0} loss {1:5.2f} | {0} ppl {2:9.3f} """.format(SCREAMING_SNAKE_CASE__       ,						SCREAMING_SNAKE_CASE__       ,						math.exp(SCREAMING_SNAKE_CASE__       )       )
  return log_str

 UpperCAmelCase__										=    """"""
 if valid_loss is not None:
  log_str += format_log(SCREAMING_SNAKE_CASE__       ,						"""valid"""       )
 if test_loss is not None:
  log_str += format_log(SCREAMING_SNAKE_CASE__       ,						"""test"""       )

 logger.info("""=""" * 100       )
 logger.info(SCREAMING_SNAKE_CASE__       )
 logger.info("""=""" * 100       )


if __name__ == "__main__":
    main()






'''simple docstring'''




import argparse

from transformers import TaConfig, TaForConditionalGeneration, load_tf_weights_in_ta
from transformers.utils import logging


logging.set_verbosity_info()

def    _UpperCamelCase      (				SCREAMING_SNAKE_CASE__			:      List[str]       ,						SCREAMING_SNAKE_CASE__			:      str       ,						SCREAMING_SNAKE_CASE__			:      Tuple       ):
 '''simple docstring'''





 UpperCAmelCase__										=    TaConfig.from_json_file(SCREAMING_SNAKE_CASE__       )
 print(F'''Building PyTorch model from configuration: {config}'''       )
 UpperCAmelCase__										=    TaForConditionalGeneration(SCREAMING_SNAKE_CASE__       )

 # Load weights from tf checkpoint
 load_tf_weights_in_ta(SCREAMING_SNAKE_CASE__       ,						SCREAMING_SNAKE_CASE__       ,						SCREAMING_SNAKE_CASE__       )

 # Save pytorch-model
 print(F'''Save PyTorch model to {pytorch_dump_path}'''       )
 model.save_pretrained(SCREAMING_SNAKE_CASE__       )


if __name__ == "__main__":
    UpperCAmelCase_       				=      argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
    )
    parser.add_argument(
        '--config_file',
        default=None,
        type=str,
        required=True,
        help=(
            'The config json file corresponding to the pre-trained T5 model. \nThis specifies the model architecture.'
        ),
    )
    parser.add_argument(
        '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
    )
    UpperCAmelCase_       				=      parser.parse_args()
    convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path)
