




"""simple docstring"""







import argparse

import torch
from saprl.hub import distilhubert

from transformers import HubertConfig, HubertModel, WavaVecaFeatureExtractor, logging


logging.set_verbosity_info()
SCREAMING_SNAKE_CASE_      =					logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_      =					{
    'post_extract_proj': 'feature_projection.projection',
    'encoder.pos_conv.0': 'encoder.pos_conv_embed.conv',
    'self_attn.k_proj': 'encoder.layers.*.attention.k_proj',
    'self_attn.v_proj': 'encoder.layers.*.attention.v_proj',
    'self_attn.q_proj': 'encoder.layers.*.attention.q_proj',
    'self_attn.out_proj': 'encoder.layers.*.attention.out_proj',
    'self_attn_layer_norm': 'encoder.layers.*.layer_norm',
    'fc1': 'encoder.layers.*.feed_forward.intermediate_dense',
    'fc2': 'encoder.layers.*.feed_forward.output_dense',
    'final_layer_norm': 'encoder.layers.*.final_layer_norm',
    'encoder.layer_norm': 'encoder.layer_norm',
    'mask_emb': 'masked_spec_embed',
}





def     __snake_case   (							_lowercase ,_lowercase ,_lowercase ,_lowercase ,_lowercase						):


    """simple docstring"""







    for attribute in key.split('''.'''						):
        UpperCamelCase										=  getattr(_lowercase ,_lowercase						)

    if weight_type is not None:
        UpperCamelCase										=  getattr(_lowercase ,_lowercase						).shape
    else:
        UpperCamelCase										=  hf_pointer.shape

    assert hf_shape == value.shape, (
        f'Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be'
        f' {value.shape} for {full_name}'
    )

    if weight_type == "weight":
        UpperCamelCase										=  value
    elif weight_type == "weight_g":
        UpperCamelCase										=  value
    elif weight_type == "weight_v":
        UpperCamelCase										=  value
    elif weight_type == "bias":
        UpperCamelCase										=  value
    else:
        UpperCamelCase										=  value

    logger.info(f'{key + "." + weight_type if weight_type is not None else ""} was initialized from {full_name}.'						)





def     __snake_case   (							_lowercase ,_lowercase						):


    """simple docstring"""







    UpperCamelCase										=  []
    UpperCamelCase										=  fairseq_model.state_dict()

    UpperCamelCase										=  hf_model.feature_extractor

    for name, value in fairseq_dict.items():
        UpperCamelCase										=  False
        if "conv_layers" in name:
            load_conv_layer(
                _lowercase ,_lowercase ,_lowercase ,_lowercase ,hf_model.config.feat_extract_norm == '''group''' ,)
            UpperCamelCase										=  True
        else:
            for key, mapped_key in MAPPING.items():
                UpperCamelCase										=  mapped_key

                if key in name:
                    UpperCamelCase										=  True
                    if "*" in mapped_key:
                        UpperCamelCase										=  name.split(_lowercase						)[0].split('''.'''						)[-2]
                        UpperCamelCase										=  mapped_key.replace('''*''' ,_lowercase						)
                    if "weight_g" in name:
                        UpperCamelCase										=  '''weight_g'''
                    elif "weight_v" in name:
                        UpperCamelCase										=  '''weight_v'''
                    elif "weight" in name:
                        UpperCamelCase										=  '''weight'''
                    elif "bias" in name:
                        UpperCamelCase										=  '''bias'''
                    else:
                        UpperCamelCase										=  None
                    set_recursively(_lowercase ,_lowercase ,_lowercase ,_lowercase ,_lowercase						)
                continue
        if not is_used:
            unused_weights.append(_lowercase						)

    logger.warning(f'Unused weights: {unused_weights}'						)





def     __snake_case   (							_lowercase ,_lowercase ,_lowercase ,_lowercase ,_lowercase						):


    """simple docstring"""







    UpperCamelCase										=  full_name.split('''conv_layers.'''						)[-1]
    UpperCamelCase										=  name.split('''.'''						)
    UpperCamelCase										=  int(items[0]						)
    UpperCamelCase										=  int(items[1]						)

    if type_id == 0:
        if "bias" in name:
            assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
                f'{full_name} has size {value.shape}, but'
                f' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'
            )
            UpperCamelCase										=  value
            logger.info(f'Feat extract conv layer {layer_id} was initialized from {full_name}.'						)
        elif "weight" in name:
            assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
                f'{full_name} has size {value.shape}, but'
                f' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'
            )
            UpperCamelCase										=  value
            logger.info(f'Feat extract conv layer {layer_id} was initialized from {full_name}.'						)
    elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
        if "bias" in name:
            assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
                f'{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was'
                " found."
            )
            UpperCamelCase										=  value
            logger.info(f'Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'						)
        elif "weight" in name:
            assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
                f'{full_name} has size {value.shape}, but'
                f' {feature_extractor[layer_id].layer_norm.weight.data.shape} was found.'
            )
            UpperCamelCase										=  value
            logger.info(f'Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'						)
    else:
        unused_weights.append(_lowercase						)





def     __snake_case   (							_lowercase						):


    """simple docstring"""







    UpperCamelCase										=  HubertConfig()
    UpperCamelCase										=  model.config

    UpperCamelCase										=  fs_config.activation_dropout
    UpperCamelCase										=  False
    UpperCamelCase										=  fs_config.attention_dropout
    UpperCamelCase										=  False
    UpperCamelCase										=  eval(fs_config.extractor_conv_feature_layers						)
    UpperCamelCase										=  [x[0] for x in conv_layers]
    UpperCamelCase										=  [x[1] for x in conv_layers]
    UpperCamelCase										=  [x[2] for x in conv_layers]
    UpperCamelCase										=  '''gelu'''
    UpperCamelCase										=  '''layer''' if fs_config.extractor_mode == '''layer_norm''' else '''group'''
    UpperCamelCase										=  False
    UpperCamelCase										=  0.0
    UpperCamelCase										=  0.0
    UpperCamelCase										=  fs_config.activation_fn
    UpperCamelCase										=  fs_config.dropout
    UpperCamelCase										=  fs_config.encoder_embed_dim
    UpperCamelCase										=  0.02
    UpperCamelCase										=  fs_config.encoder_ffn_embed_dim
    UpperCamelCase										=  1e-5
    UpperCamelCase										=  0.0
    UpperCamelCase										=  fs_config.encoder_attention_heads
    UpperCamelCase										=  fs_config.conv_pos_groups
    UpperCamelCase										=  fs_config.conv_pos
    UpperCamelCase										=  len(_lowercase						)
    UpperCamelCase										=  fs_config.encoder_layers

    return config





@torch.no_grad()
def     __snake_case   (							_lowercase ,_lowercase=None						):


    """simple docstring"""







    UpperCamelCase										=  distilhubert().model.model

    if config_path is not None:
        UpperCamelCase										=  HubertConfig.from_pretrained(_lowercase						)
    else:
        UpperCamelCase										=  convert_config(_lowercase						)
    UpperCamelCase										=  model.eval()

    UpperCamelCase										=  WavaVecaFeatureExtractor(
        feature_size=1 ,sampling_rate=1_6000 ,padding_value=0 ,do_normalize=_lowercase ,return_attention_mask=_lowercase ,)
    UpperCamelCase										=  HubertModel(_lowercase						)

    recursively_load_weights(_lowercase ,_lowercase						)

    feature_extractor.save_pretrained(_lowercase						)
    hf_model.save_pretrained(_lowercase						)


if __name__ == "__main__":
  SCREAMING_SNAKE_CASE_      =					argparse.ArgumentParser()
  parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
  parser.add_argument('--config_path', default=None, type=str, help='Path to hf config.json of model to convert')
  SCREAMING_SNAKE_CASE_      =					parser.parse_args()
  convert_hubert_checkpoint(args.pytorch_dump_folder_path, args.config_path)





"""simple docstring"""







import os
import sys
from pathlib import Path

import pytest

from datasets import Dataset, IterableDataset
from datasets.distributed import split_dataset_by_node

from .utils import execute_subprocess_async, get_torch_dist_unique_port, require_torch





def     __snake_case   (							):


    """simple docstring"""







    UpperCamelCase										=  Dataset.from_dict({'''i''': range(17						)}						)
    UpperCamelCase										=  len(_lowercase						)
    UpperCamelCase										=  3
    UpperCamelCase										=  [
        split_dataset_by_node(_lowercase ,rank=_lowercase ,world_size=_lowercase						) for rank in range(_lowercase						)
    ]
    assert sum(len(_lowercase						) for ds in datasets_per_rank						) == full_size
    assert len({tuple(x.values()						) for ds in datasets_per_rank for x in ds}						) == full_size





def     __snake_case   (							):


    """simple docstring"""







    def gen():
        return ({"i": i} for i in range(17						))

    UpperCamelCase										=  3
    UpperCamelCase										=  IterableDataset.from_generator(_lowercase						)
    UpperCamelCase										=  len(list(_lowercase						)						)
    UpperCamelCase										=  [
        split_dataset_by_node(_lowercase ,rank=_lowercase ,world_size=_lowercase						) for rank in range(_lowercase						)
    ]
    assert sum(len(list(_lowercase						)						) for ds in datasets_per_rank						) == full_size
    assert len({tuple(x.values()						) for ds in datasets_per_rank for x in ds}						) == full_size





@pytest.mark.parametrize('''shards_per_node''' ,[1, 2, 3]						)
def     __snake_case   (							_lowercase						):


    """simple docstring"""







    def gen(_lowercase						):
        for shard in shards:
            yield from ({"i": i, "shard": shard} for i in range(17						))

    UpperCamelCase										=  3
    UpperCamelCase										=  shards_per_node * world_size
    UpperCamelCase										=  {'''shards''': [f'shard_{shard_idx}.txt' for shard_idx in range(_lowercase						)]}
    UpperCamelCase										=  IterableDataset.from_generator(_lowercase ,gen_kwargs=_lowercase						)
    UpperCamelCase										=  len(list(_lowercase						)						)
    assert full_ds.n_shards == world_size * shards_per_node
    UpperCamelCase										=  [
        split_dataset_by_node(_lowercase ,rank=_lowercase ,world_size=_lowercase						) for rank in range(_lowercase						)
    ]
    assert [ds.n_shards for ds in datasets_per_rank] == [shards_per_node] * world_size
    assert sum(len(list(_lowercase						)						) for ds in datasets_per_rank						) == full_size
    assert len({tuple(x.values()						) for ds in datasets_per_rank for x in ds}						) == full_size





def     __snake_case   (							):


    """simple docstring"""







    def gen():
        return ({"i": i} for i in range(17						))

    UpperCamelCase										=  2
    UpperCamelCase										=  IterableDataset.from_generator(_lowercase						)
    UpperCamelCase										=  len(list(_lowercase						)						)

    UpperCamelCase										=  split_dataset_by_node(_lowercase ,rank=0 ,world_size=_lowercase						).shuffle(seed=42						)
    assert len(list(_lowercase						)						) == 1 + full_size // world_size
    with pytest.raises(_lowercase						):
        split_dataset_by_node(_lowercase ,rank=0 ,world_size=_lowercase						).shuffle()

    UpperCamelCase										=  split_dataset_by_node(full_ds.shuffle(seed=42						) ,rank=0 ,world_size=_lowercase						)
    assert len(list(_lowercase						)						) == 1 + full_size // world_size
    with pytest.raises(_lowercase						):
        split_dataset_by_node(full_ds.shuffle() ,rank=0 ,world_size=_lowercase						)





@pytest.mark.parametrize('''streaming''' ,[False, True]						)
@require_torch
@pytest.mark.skipif(os.name == '''nt''' ,reason='''execute_subprocess_async doesn\'t support windows'''						)
@pytest.mark.integration
def     __snake_case   (							_lowercase						):


    """simple docstring"""







    UpperCamelCase										=  2
    UpperCamelCase										=  get_torch_dist_unique_port()
    UpperCamelCase										=  Path(_lowercase						).resolve().parent / '''distributed_scripts''' / '''launch_torch_distributed.py'''
    UpperCamelCase										=  f'\n        -m torch.distributed.launch\n        --nproc_per_node={nproc_per_node}\n        --master_port={master_port}\n        {test_script}\n    '.split()
    UpperCamelCase										=  f'\n        --streaming={streaming}\n    '.split()
    UpperCamelCase										=  [sys.executable] + distributed_args + args
    execute_subprocess_async(_lowercase ,env=os.environ.copy()						)





@pytest.mark.parametrize(
    '''nproc_per_node, num_workers''' ,[
        (2, 2),  # each node has 2 shards and each worker has 1 shards
        (3, 2),  # each node uses all the shards but skips examples, and each worker has 2 shards
    ] ,)
@require_torch
@pytest.mark.skipif(os.name == '''nt''' ,reason='''execute_subprocess_async doesn\'t support windows'''						)
@pytest.mark.integration
def     __snake_case   (							_lowercase ,_lowercase						):


    """simple docstring"""







    UpperCamelCase										=  True
    UpperCamelCase										=  get_torch_dist_unique_port()
    UpperCamelCase										=  Path(_lowercase						).resolve().parent / '''distributed_scripts''' / '''launch_torch_distributed.py'''
    UpperCamelCase										=  f'\n        -m torch.distributed.launch\n        --nproc_per_node={nproc_per_node}\n        --master_port={master_port}\n        {test_script}\n    '.split()
    UpperCamelCase										=  f'\n        --streaming={streaming}\n        --num_workers={num_workers}\n    '.split()
    UpperCamelCase										=  [sys.executable] + distributed_args + args
    execute_subprocess_async(_lowercase ,env=os.environ.copy()						)