




"""simple docstring"""







import collections

from .utils import ExplicitEnum, is_torch_available, logging


if is_torch_available():
    import torch


SCREAMING_SNAKE_CASE_      =					logging.get_logger(__name__)

class 							snake_case_ :







   """simple docstring"""




   def __init__(		self					,				lowerCamelCase_					,				lowerCamelCase_=2_1					,				lowerCamelCase_=[]					,				lowerCamelCase_=None)							->      Dict:
       UpperCamelCase										=  model
       UpperCamelCase										=  trace_batch_nums
       UpperCamelCase										=  abort_after_batch_num

       # keep a LIFO buffer of frames to dump as soon as inf/nan is encountered to give context to the problem emergence
       UpperCamelCase										=  collections.deque([]					,				_lowerCAmelCase)
       UpperCamelCase										=  []
       UpperCamelCase										=  0
       UpperCamelCase										=  0
       UpperCamelCase										=  False
       UpperCamelCase										=  """                 """

       self.analyse_model()

       self.register_forward_hook()







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_=None)							->      List[Any]:
       if frame is not None:
           self.expand_frame(_lowerCAmelCase)
       self.frames.append('''\n'''.join(self.frame))
       UpperCamelCase										=  []  # start a new frame







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_)							->      List[Any]:
       self.frame.append(_lowerCAmelCase)







   def      UpperCAmelCase__ (		self)							->      Dict:
       print('''\n'''.join(self.frames))
       UpperCamelCase										=  []







   def      UpperCAmelCase__ (		self)							->      Tuple:
       UpperCamelCase										=  []







   def      UpperCAmelCase__ (		self)							->      str:
       print(F'\nDetected inf/nan during batch_number={self.batch_number}')
       print(F'Last {len(self.frames)} forward frames:')
       print(F'{"abs min":8} {"abs max":8} metadata')
       print('''\n'''.join(self.frames))
       print('''\n\n''')
       UpperCamelCase										=  []







   def      UpperCAmelCase__ (		self)							->      Union[str, Any]:
       # extract the fully qualified module names, to be able to report at run time. e.g.:
       # encoder.block.2.layer.0.SelfAttention.o
       #
       # for shared weights only the first shared module name will be registered
       UpperCamelCase										=  {m: name for name, m in self.model.named_modules()}
       # self.longest_module_name = max(len(v) for v in self.module_names.values())







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_					,				lowerCamelCase_)							->      Optional[int]:
       if torch.is_tensor(_lowerCAmelCase):
           self.expand_frame(get_abs_min_max(_lowerCAmelCase					,				_lowerCAmelCase))
           if detect_overflow(_lowerCAmelCase					,				_lowerCAmelCase):
               UpperCamelCase										=  True
       elif var is None:
           self.expand_frame(F'{"None":>17} {ctx}')
       else:
           self.expand_frame(F'{"not a tensor":>17} {ctx}')







   def      UpperCAmelCase__ (		self)							->      Dict:
       self.expand_frame(F'\n\n{self.prefix} *** Starting batch number={self.batch_number} ***')
       self.expand_frame(F'{"abs min":8} {"abs max":8} metadata')







   def      UpperCAmelCase__ (		self)							->      Dict:
       self.expand_frame(F'{self.prefix} *** Finished batch number={self.batch_number-1} ***\n\n')







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_)							->      Optional[Any]:
       self.expand_frame(F'{self.prefix} {self.module_names[module]} {module.__class__.__name__}')

       # params
       for name, p in module.named_parameters(recurse=_lowerCAmelCase):
           self.analyse_variable(_lowerCAmelCase					,				_lowerCAmelCase)

       # inputs
       if isinstance(_lowerCAmelCase					,				_lowerCAmelCase):
           for i, x in enumerate(_lowerCAmelCase):
               self.analyse_variable(_lowerCAmelCase					,				F'input[{i}]')
       else:
           self.analyse_variable(_lowerCAmelCase					,				'''input''')

       # outputs
       if isinstance(_lowerCAmelCase					,				_lowerCAmelCase):
           for i, x in enumerate(_lowerCAmelCase):
               # possibly a tuple of tuples
               if isinstance(_lowerCAmelCase					,				_lowerCAmelCase):
                   for j, y in enumerate(_lowerCAmelCase):
                       self.analyse_variable(_lowerCAmelCase					,				F'output[{i}][{j}]')
               else:
                   self.analyse_variable(_lowerCAmelCase					,				F'output[{i}]')
       else:
           self.analyse_variable(_lowerCAmelCase					,				'''output''')

       self.save_frame()







   def      UpperCAmelCase__ (		self)							->      Any:
       self.model.apply(self._register_forward_hook)







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_)							->      Optional[int]:
       module.register_forward_hook(self.forward_hook)







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_)							->      List[str]:
       # - input is a tuple of packed inputs (could be non-Tensors)
       # - output could be a Tensor or a tuple of Tensors and non-Tensors

       UpperCamelCase										=  False

       UpperCamelCase										=  True if self.batch_number in self.trace_batch_nums else False
       if trace_mode:
           self.reset_saved_frames()

       if self.total_calls == 0:
           self.batch_start_frame()
       self.total_calls += 1

       # count batch numbers - the very first forward hook of the batch will be called when the
       # batch completes - i.e. it gets called very last - we know this batch has finished
       if module == self.model:
           self.batch_number += 1
           UpperCamelCase										=  True

       self.create_frame(_lowerCAmelCase					,				_lowerCAmelCase					,				_lowerCAmelCase)

       # if last_frame_of_batch:
       #     self.batch_end_frame()

       if trace_mode:
           self.trace_frames()

       if last_frame_of_batch:
           self.batch_start_frame()

       if self.detected_overflow and not trace_mode:
           self.dump_saved_frames()

           # now we can abort, as it's pointless to continue running
           raise ValueError(
               '''DebugUnderflowOverflow: inf/nan detected, aborting as there is no point running further. '''
               '''Please scroll up above this traceback to see the activation values prior to this event.''')

       # abort after certain batch if requested to do so
       if self.abort_after_batch_num is not None and self.batch_number > self.abort_after_batch_num:
           raise ValueError(
               F'DebugUnderflowOverflow: aborting after {self.batch_number} batches due to'
               F' `abort_after_batch_num={self.abort_after_batch_num}` arg')





def     __snake_case   (							_lowercase ,_lowercase						):


    """simple docstring"""







    UpperCamelCase										=  var.abs()
    return f'{abs_var.min():8.2e} {abs_var.max():8.2e} {ctx}'





def     __snake_case   (							_lowercase ,_lowercase						):


    """simple docstring"""







    UpperCamelCase										=  False
    if torch.isnan(__lowerCamelCase						).any().item():
        UpperCamelCase										=  True
        print(f'{ctx} has nans'						)
    if torch.isinf(__lowerCamelCase						).any().item():
        UpperCamelCase										=  True
        print(f'{ctx} has infs'						)

    # if needed to monitor large elements can enable the following
    if 0:  # and detected:
        UpperCamelCase										=  var[torch.ge(var.abs() ,100						)]
        if naaa.numel() > 0:
            print(f'{ctx}:  n100={naaa.numel()}'						)
        UpperCamelCase										=  var[torch.ge(var.abs() ,1000						)]
        if naaaa.numel() > 0:
            print(f'{ctx}: n1000={naaaa.numel()}'						)
        UpperCamelCase										=  var[torch.ge(var.abs() ,1_0000						)]
        if naaaaa.numel() > 0:
            print(f'{ctx}: n10000={naaaaa.numel()}'						)

    if 0:
        print(f'min={var.min():9.2e} max={var.max():9.2e}'						)

    if 0:
        print(f'min={var.min():9.2e} max={var.max():9.2e} var={var.var():9.2e} mean={var.mean():9.2e} ({ctx})'						)

    return detected


class 							snake_case_ (				__lowerCamelCase ):







   """simple docstring"""




   A_										=					'''underflow_overflow'''
   A_										=					'''tpu_metrics_debug'''





"""simple docstring"""







import os
import sys
from pathlib import Path

import pytest

from datasets import Dataset, IterableDataset
from datasets.distributed import split_dataset_by_node

from .utils import execute_subprocess_async, get_torch_dist_unique_port, require_torch





def     __snake_case   (							):


    """simple docstring"""







    UpperCamelCase										=  Dataset.from_dict({'''i''': range(17						)}						)
    UpperCamelCase										=  len(_lowercase						)
    UpperCamelCase										=  3
    UpperCamelCase										=  [
        split_dataset_by_node(_lowercase ,rank=_lowercase ,world_size=_lowercase						) for rank in range(_lowercase						)
    ]
    assert sum(len(_lowercase						) for ds in datasets_per_rank						) == full_size
    assert len({tuple(x.values()						) for ds in datasets_per_rank for x in ds}						) == full_size





def     __snake_case   (							):


    """simple docstring"""







    def gen():
        return ({"i": i} for i in range(17						))

    UpperCamelCase										=  3
    UpperCamelCase										=  IterableDataset.from_generator(_lowercase						)
    UpperCamelCase										=  len(list(_lowercase						)						)
    UpperCamelCase										=  [
        split_dataset_by_node(_lowercase ,rank=_lowercase ,world_size=_lowercase						) for rank in range(_lowercase						)
    ]
    assert sum(len(list(_lowercase						)						) for ds in datasets_per_rank						) == full_size
    assert len({tuple(x.values()						) for ds in datasets_per_rank for x in ds}						) == full_size





@pytest.mark.parametrize('''shards_per_node''' ,[1, 2, 3]						)
def     __snake_case   (							_lowercase						):


    """simple docstring"""







    def gen(_lowercase						):
        for shard in shards:
            yield from ({"i": i, "shard": shard} for i in range(17						))

    UpperCamelCase										=  3
    UpperCamelCase										=  shards_per_node * world_size
    UpperCamelCase										=  {'''shards''': [f'shard_{shard_idx}.txt' for shard_idx in range(_lowercase						)]}
    UpperCamelCase										=  IterableDataset.from_generator(_lowercase ,gen_kwargs=_lowercase						)
    UpperCamelCase										=  len(list(_lowercase						)						)
    assert full_ds.n_shards == world_size * shards_per_node
    UpperCamelCase										=  [
        split_dataset_by_node(_lowercase ,rank=_lowercase ,world_size=_lowercase						) for rank in range(_lowercase						)
    ]
    assert [ds.n_shards for ds in datasets_per_rank] == [shards_per_node] * world_size
    assert sum(len(list(_lowercase						)						) for ds in datasets_per_rank						) == full_size
    assert len({tuple(x.values()						) for ds in datasets_per_rank for x in ds}						) == full_size





def     __snake_case   (							):


    """simple docstring"""







    def gen():
        return ({"i": i} for i in range(17						))

    UpperCamelCase										=  2
    UpperCamelCase										=  IterableDataset.from_generator(_lowercase						)
    UpperCamelCase										=  len(list(_lowercase						)						)

    UpperCamelCase										=  split_dataset_by_node(_lowercase ,rank=0 ,world_size=_lowercase						).shuffle(seed=42						)
    assert len(list(_lowercase						)						) == 1 + full_size // world_size
    with pytest.raises(_lowercase						):
        split_dataset_by_node(_lowercase ,rank=0 ,world_size=_lowercase						).shuffle()

    UpperCamelCase										=  split_dataset_by_node(full_ds.shuffle(seed=42						) ,rank=0 ,world_size=_lowercase						)
    assert len(list(_lowercase						)						) == 1 + full_size // world_size
    with pytest.raises(_lowercase						):
        split_dataset_by_node(full_ds.shuffle() ,rank=0 ,world_size=_lowercase						)





@pytest.mark.parametrize('''streaming''' ,[False, True]						)
@require_torch
@pytest.mark.skipif(os.name == '''nt''' ,reason='''execute_subprocess_async doesn\'t support windows'''						)
@pytest.mark.integration
def     __snake_case   (							_lowercase						):


    """simple docstring"""







    UpperCamelCase										=  2
    UpperCamelCase										=  get_torch_dist_unique_port()
    UpperCamelCase										=  Path(_lowercase						).resolve().parent / '''distributed_scripts''' / '''launch_torch_distributed.py'''
    UpperCamelCase										=  f'\n        -m torch.distributed.launch\n        --nproc_per_node={nproc_per_node}\n        --master_port={master_port}\n        {test_script}\n    '.split()
    UpperCamelCase										=  f'\n        --streaming={streaming}\n    '.split()
    UpperCamelCase										=  [sys.executable] + distributed_args + args
    execute_subprocess_async(_lowercase ,env=os.environ.copy()						)





@pytest.mark.parametrize(
    '''nproc_per_node, num_workers''' ,[
        (2, 2),  # each node has 2 shards and each worker has 1 shards
        (3, 2),  # each node uses all the shards but skips examples, and each worker has 2 shards
    ] ,)
@require_torch
@pytest.mark.skipif(os.name == '''nt''' ,reason='''execute_subprocess_async doesn\'t support windows'''						)
@pytest.mark.integration
def     __snake_case   (							_lowercase ,_lowercase						):


    """simple docstring"""







    UpperCamelCase										=  True
    UpperCamelCase										=  get_torch_dist_unique_port()
    UpperCamelCase										=  Path(_lowercase						).resolve().parent / '''distributed_scripts''' / '''launch_torch_distributed.py'''
    UpperCamelCase										=  f'\n        -m torch.distributed.launch\n        --nproc_per_node={nproc_per_node}\n        --master_port={master_port}\n        {test_script}\n    '.split()
    UpperCamelCase										=  f'\n        --streaming={streaming}\n        --num_workers={num_workers}\n    '.split()
    UpperCamelCase										=  [sys.executable] + distributed_args + args
    execute_subprocess_async(_lowercase ,env=os.environ.copy()						)