light files

mimic3_make_harvard_sentences.py

@@ -1,3 +1,4 @@
+# https://github.com/audeering/shift/tree/main  -- RUN FROM THIS REPO
 import shutil
 import csv
 import io
@@ -62,12 +63,12 @@ import audiofile
 # [print(i) for i in foreign_voices]
 # print('\n_______________________________\n')
 # [print(i) for i in english_voices]
-# ====================================================== END PRINT LIST OF VOICES
+# ====================================================== LIST Mimic-3 ALL VOICES
 list_voices = [
     'en_US/m-ailabs_low#mary_ann',
     'en_UK/apope_low',
     'de_DE/thorsten-emotion_low#neutral',  # is the 4x really interesting we can just write it in Section
-    'human'
+    'human',
     ]  # special - for human we load specific style file - no Mimic3 is run
 
 
@@ -290,7 +291,7 @@ for _id, _voice in enumerate(list_voices):
         with open('harvard.json', 'r') as f:
             harvard_individual_sentences = json.load(f)['sentences']
         total_audio_mimic3 = []
-        total_audio_stts2 = []
+        total_audio_styletts2 = []
         ix = 0
         for list_of_10 in harvard_individual_sentences[:1]:  # 77
             
@@ -341,16 +342,22 @@ for _id, _voice in enumerate(list_voices):
                 # # state.ssml = 1234546575
                 # state.stdout = True
                 # state.tts = True
-                process_lines(state, wav_path='tmp1.wav')
+                style_path = 'tmp1.wav'
+                process_lines(state, wav_path=style_path)
                 shutdown_tts(state)
-                x, fs = audiofile.read('tmp1.wav')
-                print(x.shape)
+                x, fs = audiofile.read(style_path)
+                # print(x.shape)
             else:
+                # --
                 # MSP['valence.train.votes'].get().sort_values('7').index[-1]
-                human_style = '/cache/audb/msppodcast/2.4.0/fe182b91/Audios/MSP-PODCAST_0235_0053.wav'
-                x, fs = audiofile.read(human_style)
+                # style_path = '/cache/audb/msppodcast/2.4.0/fe182b91/Audios/MSP-PODCAST_0235_0053.wav'
+                # --
+                # MSP['emotion.test-1'].get().sort_values('valence').index[-1]
+                style_path = '/cache/audb/msppodcast/2.4.0/fe182b91/Audios/MSP-PODCAST_0220_0870.wav'
+                x, fs = audiofile.read(style_path)  # assure is not very short - equl harvard sent len
                 print(x.shape,' human')   # crop human to almost mimic-3 duration
             total_audio_mimic3.append(x)
+            print(f'{len(total_audio_mimic3)=}')
             print(fs, text, 'mimic3')
             
             # MIMIC3 = = = = = = = = = = = = = = END
@@ -358,7 +365,7 @@ for _id, _voice in enumerate(list_voices):
 
 
             
-            style_vec = msinference.compute_style('tmp1.wav')  # use mimic-3 as prompt
+            style_vec = msinference.compute_style(style_path)  # use mimic-3 as prompt
             
 
 
@@ -369,39 +376,47 @@ for _id, _voice in enumerate(list_voices):
                                         diffusion_steps=7,
                                         embedding_scale=1)
             
-            total_audio_stts2.append(x)
+            total_audio_styletts2.append(x)
+ 
+        # save styletts2 .wav
 
+        total_audio_styletts2 = np.concatenate(total_audio_styletts2) # -- concat 77x lists
+        total_audio_styletts2 = audresample.resample(total_audio_styletts2, 
+                                                     original_rate=24000,
+                                                     target_rate=16000)[0]
+        print('RESAMPLEstyletts2', total_audio_styletts2.shape)
+        audiofile.write(out_dir + 'styletts2__' + _str + '.wav', total_audio_styletts2, 16000)
+        # print('Saving:', out_dir + 'styletts2__' + _str + '.wav')
 
-
-
-
-
-        total_audio_stts2 = np.concatenate(total_audio_stts2) # -- concat 77x lists
-        total_audio_stts2 = audresample.resample(total_audio_stts2, original_rate=24000, target_rate=16000)[0]  # for audinterface
-        audiofile.write(out_dir + 'styletts2__' + _str + '.wav', total_audio_stts2, 16000)
+        # save mimic3 or human .wav
 
         total_audio_mimic3 = np.concatenate(total_audio_mimic3) # -- concat 77x lists
-        total_audio_mimic3 = audresample.resample(total_audio_mimic3, original_rate=24000, target_rate=16000)[0]
+        if 'human' not in _str:
+            total_audio_mimic3 = audresample.resample(total_audio_mimic3,
+                                                      original_rate=24000, 
+                                                      target_rate=16000)[0]
+        else:
+            print('human is already on 16kHz - MSPpodcst file')
+        print('RESAMPLEmimic3', total_audio_mimic3.shape)
         audiofile.write(out_dir + 'mimic3__' + _str + '.wav', total_audio_mimic3, 16000)
 
-        print('Saving:', out_dir + 'mimic3__' + _str + '.wav')
-    else:
-        print('Skip:', out_dir + 'styletts2__' + _str + '.wav')
+        print(total_audio_mimic3.shape, total_audio_styletts2.shape, 'LEN OF TOTAL\n')
+        # print('Saving:', out_dir + 'mimic3__' + _str + '.wav')
 
         
     # AUD   I N T E R F A C E
-        # file_interface = f'timeseries_{long_audio.replace("/", "")}.pkl'
+    
 
 
 
-    for engine in ['mimic3', 'styletts2']:
+    for engine in ['mimic3',
+                   'styletts2']:
         harvard_of_voice = f'{out_dir}{engine}__{_str}'
         if not os.path.exists(harvard_of_voice + '.pkl'):
             df = interface.process_file(harvard_of_voice + '.wav')
             df.to_pickle(harvard_of_voice + '.pkl')
-        else:
-            # df = pd.read_pickle(harvard_of_voice + '.pkl')
-            print(harvard_of_voice + '.pkl', 'FOUND')
+            print('\n\n', harvard_of_voice, df,'\n___________________________\n')
+
 
 
 
@@ -411,150 +426,203 @@ for _id, _voice in enumerate(list_voices):
 
         
         
-
+raise SystemExit
 print('\nVisuals\n')
 
 # ===============================================================================
 # V I S U A L S
 #
 # ===============================================================================
+voice_pairs = [
+    [list_voices[0], list_voices[1]],
+    [list_voices[2], list_voices[3]]
+    ]  # special - for human we load specific style file - no Mimic3 is run
 
-for folder, list_voices in [
-        ['foreign', foreign_voices],
-        ['english', english_voices],
-            ]:
-    print(folder, list_voices[:4], '\n\nVISUALIZING VOICES')                           
-    for _id, _voice in enumerate(list_voices[:4]):
-        _str = _voice.replace('/', '_').replace('#', '_').replace('_low', '')
-        _dir = folder + '_pkl/'
-        if 'cmu-arctic' in _str:
-            _str = _str.replace('cmu-arctic', 'cmu_arctic') #+ '.wav'
-                
-        
-        vis_df = {}
-        # LOAD PKL
-        for engine in ['mimic3', 'styletts2']:
-            harvard_of_voice = f'{_dir}{engine}__{_str}'
-            if not os.path.exists(harvard_of_voice + '.pkl'):
-                df = interface.process_file(harvard_of_voice + '.wav')
-                df.to_pickle(harvard_of_voice + '.pkl')
-            else:
-                df = pd.read_pickle(harvard_of_voice + '.pkl')
-                print(harvard_of_voice + '.pkl', 'FOUND')
+# PLot 1   list_voices[0] list_voices[1]
+# Plot 2   list_voices[2] list_voices[2]
+
+for vox1, vox2 in voice_pairs:  # 1 figure pro pair
+
+    _str1 = vox1.replace('/', '_').replace('#', '_').replace('_low', '')
+    
+    if 'cmu-arctic' in _str1:
+        _str1 = _str1.replace('cmu-arctic', 'cmu_arctic') #+ '.wav'
+
+    _str2 = vox2.replace('/', '_').replace('#', '_').replace('_low', '')
+    
+    if 'cmu-arctic' in _str2:
+        _str2 = _str2.replace('cmu-arctic', 'cmu_arctic') #+ '.wav'
 
-            vis_df[engine] = df
-        SHORT = min(len(vis_df['mimic3']), len(vis_df['styletts2']))
-        for k,v in vis_df.items():
-            p = v[:SHORT]  # TRuncate extra segments - human is slower than mimic3
             
-            p.reset_index(inplace= True)
-            p.drop(columns=['file','start'], inplace=True)
-            p.set_index('end', inplace=True)
-            # p = p.filter(scene_classes) #['transport', 'indoor', 'outdoor'])
-            p.index = p.index.map(mapper = (lambda x: x.total_seconds()))
-            vis_df[k] = p
+    vis_df = {
+        f'mimic3_{_str1}'    : pd.read_pickle(out_dir + 'mimic3__' + _str1 + '.pkl'),
+        f'mimic3_{_str2}'    : pd.read_pickle(out_dir + 'mimic3__' + _str2 + '.pkl'),
+        f'styletts2_{_str1}' : pd.read_pickle(out_dir + 'styletts2__' + _str1 + '.pkl'),
+        f'styletts2_{_str2}' : pd.read_pickle(out_dir + 'styletts2__' + _str2 + '.pkl'),
+    }
+
 
+
+    
+    SHORT_LEN = min([len(v) for k, v in vis_df.items()])  # different TTS durations per voic
+    for k,v in vis_df.items():
+        p = v[:SHORT_LEN]  # TRuncate extra segments - human is slower than mimic3
+        print('\n\n\n\n',k, p)
+        p.reset_index(inplace= True)
+        p.drop(columns=['file','start'], inplace=True)
+        p.set_index('end', inplace=True)
+        # p = p.filter(scene_classes) #['transport', 'indoor', 'outdoor'])
+        p.index = p.index.map(mapper = (lambda x: x.total_seconds()))
+        vis_df[k] = p
+    preds = vis_df
+    fig, ax = plt.subplots(nrows=10, ncols=2, figsize=(24, 24), gridspec_kw={'hspace': 0, 'wspace': .04})
+
+
+    # ADV - subplots
+
+    time_stamp = preds[f'mimic3_{_str2}'].index.to_numpy()
+    for j, dim in enumerate(['arousal', 
+                            'dominance', 
+                            'valence']):
+
+        # MIMIC3                      
+
+        ax[j, 0].plot(time_stamp, preds[f'styletts2_{_str1}'][dim], 
+                    color=(0,104/255,139/255), 
+                    label='mean_1', 
+                    linewidth=2)
+        ax[j, 0].fill_between(time_stamp,
+
+                        preds[f'styletts2_{_str1}'][dim],
+                        preds[f'mimic3_{_str1}'][dim],
+
+                        color=(.2,.2,.2), 
+                        alpha=0.244)
+        if j == 0:                    
+            ax[j, 0].legend([f'mimic3_{_str1}',
+                            f'StyleTTS2 using {_str1}'], 
+                            prop={'size': 10}, 
+                            #  loc='lower right'
+                            )
+        ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=14)
         
-        print(vis_df, '\n\n\n\n \n')
-        # ============ VISUAL ADV cats of styletts2 vs mimic3 same-voice
+        # TICK
+        ax[j, 0].set_ylim([1e-7, .9999])
+        # ax[j, 0].set_yticks([.25, .5,.75])
+        # ax[j, 0].set_yticklabels(['0.25', '.5', '0.75'])
+        ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
+        ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]])
 
 
+    # MIMIC3   4x speed
 
-        fig, ax = plt.subplots(nrows=10, ncols=2, figsize=(24, 24), 
-                               gridspec_kw={'hspace': 0, 'wspace': .04})
 
-        
-        # ADV 
-        
+        ax[j, 1].plot(time_stamp, preds[f'mimic3_{_str2}'][dim], 
+                    color=(0,104/255,139/255), 
+                    label='mean_1', 
+                    linewidth=2)
+        ax[j, 1].fill_between(time_stamp,
 
-        time_stamp = vis_df['mimic3'].index.to_numpy()
-        for j, dim in enumerate(['arousal', 
-                                'dominance', 
-                                'valence']):
-
-            # MIMIC3                      
-
-            ax[j, 0].plot(time_stamp, vis_df['mimic3'][dim], 
-                        color=(0,104/255,139/255), 
-                        label='mean_1', 
-                        linewidth=2)
-            ax[j, 0].fill_between(time_stamp,
-
-                            vis_df['mimic3'][dim],
-                            vis_df['styletts2'][dim],
-
-                            color=(.2,.2,.2), 
-                            alpha=0.244)
-            if j == 0:                    
-                ax[j, 0].legend(['StyleTTS2 style mimic3',
-                                'StyleTTS2 style crema-d'], 
-                                prop={'size': 10}, 
-                                #  loc='lower right'
-                                )
-            ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=14)
-            
-            # TICK
-            ax[j, 0].set_ylim([1e-7, .9999])
-            # ax[j, 0].set_yticks([.25, .5,.75])
-            # ax[j, 0].set_yticklabels(['0.25', '.5', '0.75'])
-            ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
-            ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]])
+                        preds[f'styletts2_{_str2}'][dim],
+                        preds[f'mimic3_{_str2}'][dim],
 
+                        color=(.2,.2,.2), 
+                        alpha=0.244)
+        if j == 0:                    
+            ax[j, 1].legend([f'mimic3_{_str2}',
+                            f'StyleTTS2 using {_str2}'], 
+                            prop={'size': 10}, 
+                            #  loc='lower right'
+                            )
 
-            ax[j, 0].grid()
-            
-        # CATEGORIE
+
+        ax[j, 1].set_xlabel('767 Harvard Sentences (seconds)')
+
+
+
+        # TICK
+        ax[j, 1].set_ylim([1e-7, .9999])
+        # ax[j, 1].set_yticklabels(['' for _ in ax[j, 1].get_yticklabels()])
+        ax[j, 1].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
+        ax[j, 1].set_xlim([time_stamp[0], time_stamp[-1]])
+
+
+
+
+        ax[j, 0].grid()
+        ax[j, 1].grid()
+    # CATEGORIE
 
 
 
 
 
-        time_stamp = vis_df['styletts2'].index.to_numpy()
-        for j, dim in enumerate(['Angry', 
-                                'Sad',
-                                'Happy',
-                                'Surprise', 
-                                'Fear',
-                                'Disgust', 
-                                'Contempt',
-                                #  'Neutral'
-                                ]):   # ASaHSuFDCN
-            j = j + 3  # skip A/D/V suplt                         
+    time_stamp = preds[f'mimic3_{_str1}'].index.to_numpy()
+    for j, dim in enumerate(['Angry', 
+                            'Sad',
+                            'Happy',
+                            'Surprise', 
+                            'Fear',
+                            'Disgust', 
+                            'Contempt',
+                            #  'Neutral'
+                            ]):   # ASaHSuFDCN
+        j = j + 3  # skip A/D/V suplt                         
 
-            # MIMIC3                      
+        # MIMIC3                      
 
-            ax[j, 0].plot(time_stamp, vis_df['mimic3'][dim], 
-                        color=(0,104/255,139/255), 
-                        label='mean_1', 
-                        linewidth=2)
-            ax[j, 0].fill_between(time_stamp,
+        ax[j, 0].plot(time_stamp, preds[f'mimic3_{_str1}'][dim], 
+                    color=(0,104/255,139/255), 
+                    label='mean_1', 
+                    linewidth=2)
+        ax[j, 0].fill_between(time_stamp,
 
-                            vis_df['mimic3'][dim],
-                            vis_df['styletts2'][dim],
+                        preds[f'mimic3_{_str2}'][dim],
+                        preds[f'styletts2_{_str2}'][dim],
 
-                            color=(.2,.2,.2), 
-                            alpha=0.244)
-            # ax[j, 0].legend(['StyleTTS2 style mimic3',
-            #                  'StyleTTS2 style crema-d'], 
-            #                  prop={'size': 10}, 
-            #                 #  loc='upper left'
-            # )
+                        color=(.2,.2,.2), 
+                        alpha=0.244)
+        # ax[j, 0].legend(['StyleTTS2 style mimic3',
+        #                  'StyleTTS2 style crema-d'], 
+        #                  prop={'size': 10}, 
+        #                 #  loc='upper left'
+        # )
 
 
-            ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=14)
+        ax[j, 0].set_ylabel(dim.lower(), color=(.4, .4, .4), fontsize=14)
 
-            # TICKS
-            ax[j, 0].set_ylim([1e-7, .9999])
-            ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]])
-            ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
-            ax[j, 0].set_xlabel('767 Harvard Sentences (seconds)', fontsize=16, color=(.4,.4,.4))
+        # TICKS
+        ax[j, 0].set_ylim([1e-7, .9999])
+        ax[j, 0].set_xlim([time_stamp[0], time_stamp[-1]])
+        ax[j, 0].set_xticklabels(['' for _ in ax[j, 0].get_xticklabels()])
+        ax[j, 0].set_xlabel('767 Harvard Sentences (seconds)', fontsize=16, color=(.4,.4,.4))
 
 
-            ax[j, 0].grid()
+    # MIMIC3   4x speed
 
 
+        ax[j, 1].plot(time_stamp, preds[f'mimic3_{_str2}'][dim], 
+                    color=(0,104/255,139/255), 
+                    label='mean_1', 
+                    linewidth=2)
+        ax[j, 1].fill_between(time_stamp,
 
-        plt.savefig(f'bh_{_str}.png', bbox_inches='tight')
-        plt.close()
+                        preds[f'mimic3_{_str2}'][dim],
+                        preds[f'styletts2_{_str2}'][dim],
 
-print('UNCOMMENT msinfereence')
+                        color=(.2,.2,.2), 
+                        alpha=0.244)
+        # ax[j, 1].legend(['StyleTTS2 style mimic3   4x speed',
+        #                  'StyleTTS2 style crema-d'], 
+        #                  prop={'size': 10},
+        #                 #  loc='upper left'
+        # )
+        ax[j, 1].set_xlabel('767 Harvard Sentences (seconds)', fontsize=16, color=(.4,.4,.4))
+        ax[j, 1].set_ylim([1e-7, .999])
+        # ax[j, 1].set_yticklabels(['' for _ in ax[j, 1].get_yticklabels()])
+        ax[j, 1].set_xticklabels(['' for _ in ax[j, 1].get_xticklabels()])
+        ax[j, 1].set_xlim([time_stamp[0], time_stamp[-1]])
+        ax[j, 0].grid()
+        ax[j, 1].grid()
+    plt.savefig(f'pair_{_str1}_{_str2}.png', bbox_inches='tight')
+    plt.close()

models.py

@@ -0,0 +1,611 @@
+#coding:utf-8
+
+import os
+import os.path as osp
+
+import copy
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
+
+from Utils.ASR.models import ASRCNN
+from Utils.JDC.model import JDCNet
+
+from Modules.diffusion.sampler import KDiffusion, LogNormalDistribution
+from Modules.diffusion.modules import Transformer1d, StyleTransformer1d
+from Modules.diffusion.diffusion import AudioDiffusionConditional
+
+
+
+from munch import Munch
+import yaml
+
+class LearnedDownSample(nn.Module):
+    def __init__(self, layer_type, dim_in):
+        super().__init__()
+        self.layer_type = layer_type
+
+        if self.layer_type == 'none':
+            self.conv = nn.Identity()
+        elif self.layer_type == 'timepreserve':
+            self.conv = spectral_norm(nn.Conv2d(dim_in, dim_in, kernel_size=(3, 1), stride=(2, 1), groups=dim_in, padding=(1, 0)))
+        elif self.layer_type == 'half':
+            self.conv = spectral_norm(nn.Conv2d(dim_in, dim_in, kernel_size=(3, 3), stride=(2, 2), groups=dim_in, padding=1))
+        else:
+            raise RuntimeError('Got unexpected donwsampletype %s, expected is [none, timepreserve, half]' % self.layer_type)
+            
+    def forward(self, x):
+        return self.conv(x)
+
+class LearnedUpSample(nn.Module):
+    def __init__(self, layer_type, dim_in):
+        super().__init__()
+        self.layer_type = layer_type
+        
+        if self.layer_type == 'none':
+            self.conv = nn.Identity()
+        elif self.layer_type == 'timepreserve':
+            self.conv = nn.ConvTranspose2d(dim_in, dim_in, kernel_size=(3, 1), stride=(2, 1), groups=dim_in, output_padding=(1, 0), padding=(1, 0))
+        elif self.layer_type == 'half':
+            self.conv = nn.ConvTranspose2d(dim_in, dim_in, kernel_size=(3, 3), stride=(2, 2), groups=dim_in, output_padding=1, padding=1)
+        else:
+            raise RuntimeError('Got unexpected upsampletype %s, expected is [none, timepreserve, half]' % self.layer_type)
+
+
+    def forward(self, x):
+        return self.conv(x)
+
+class DownSample(nn.Module):
+    def __init__(self, layer_type):
+        super().__init__()
+        self.layer_type = layer_type
+
+    def forward(self, x):
+        if self.layer_type == 'none':
+            return x
+        elif self.layer_type == 'timepreserve':
+            return F.avg_pool2d(x, (2, 1))
+        elif self.layer_type == 'half':
+            if x.shape[-1] % 2 != 0:
+                x = torch.cat([x, x[..., -1].unsqueeze(-1)], dim=-1)
+            return F.avg_pool2d(x, 2)
+        else:
+            raise RuntimeError('Got unexpected donwsampletype %s, expected is [none, timepreserve, half]' % self.layer_type)
+
+
+class UpSample(nn.Module):
+    def __init__(self, layer_type):
+        super().__init__()
+        self.layer_type = layer_type
+
+    def forward(self, x):
+        if self.layer_type == 'none':
+            return x
+        elif self.layer_type == 'timepreserve':
+            return F.interpolate(x, scale_factor=(2, 1), mode='nearest')
+        elif self.layer_type == 'half':
+            return F.interpolate(x, scale_factor=2, mode='nearest')
+        else:
+            raise RuntimeError('Got unexpected upsampletype %s, expected is [none, timepreserve, half]' % self.layer_type)
+
+
+class ResBlk(nn.Module):
+    def __init__(self, dim_in, dim_out, actv=nn.LeakyReLU(0.2),
+                 normalize=False, downsample='none'):
+        super().__init__()
+        self.actv = actv
+        self.normalize = normalize
+        self.downsample = DownSample(downsample)
+        self.downsample_res = LearnedDownSample(downsample, dim_in)
+        self.learned_sc = dim_in != dim_out
+        self._build_weights(dim_in, dim_out)
+
+    def _build_weights(self, dim_in, dim_out):
+        self.conv1 = spectral_norm(nn.Conv2d(dim_in, dim_in, 3, 1, 1))
+        self.conv2 = spectral_norm(nn.Conv2d(dim_in, dim_out, 3, 1, 1))
+        if self.normalize:
+            self.norm1 = nn.InstanceNorm2d(dim_in, affine=True)
+            self.norm2 = nn.InstanceNorm2d(dim_in, affine=True)
+        if self.learned_sc:
+            self.conv1x1 = spectral_norm(nn.Conv2d(dim_in, dim_out, 1, 1, 0, bias=False))
+
+    def _shortcut(self, x):
+        if self.learned_sc:
+            x = self.conv1x1(x)
+        if self.downsample:
+            x = self.downsample(x)
+        return x
+
+    def _residual(self, x):
+        if self.normalize:
+            x = self.norm1(x)
+        x = self.actv(x)
+        x = self.conv1(x)
+        x = self.downsample_res(x)
+        if self.normalize:
+            x = self.norm2(x)
+        x = self.actv(x)
+        x = self.conv2(x)
+        return x
+
+    def forward(self, x):
+        x = self._shortcut(x) + self._residual(x)
+        return x / math.sqrt(2)  # unit variance
+
+class StyleEncoder(nn.Module):
+    def __init__(self, dim_in=48, style_dim=48, max_conv_dim=384):
+        super().__init__()
+        blocks = []
+        blocks += [spectral_norm(nn.Conv2d(1, dim_in, 3, 1, 1))]
+
+        repeat_num = 4
+        for _ in range(repeat_num):
+            dim_out = min(dim_in*2, max_conv_dim)
+            blocks += [ResBlk(dim_in, dim_out, downsample='half')]
+            dim_in = dim_out
+
+        blocks += [nn.LeakyReLU(0.2)]
+        blocks += [spectral_norm(nn.Conv2d(dim_out, dim_out, 5, 1, 0))]
+        blocks += [nn.AdaptiveAvgPool2d(1)]
+        blocks += [nn.LeakyReLU(0.2)]
+        self.shared = nn.Sequential(*blocks)
+
+        self.unshared = nn.Linear(dim_out, style_dim)
+
+    def forward(self, x):
+        h = self.shared(x)
+        h = h.view(h.size(0), -1)
+        s = self.unshared(h)
+    
+        return s
+
+class LinearNorm(torch.nn.Module):
+    def __init__(self, in_dim, out_dim, bias=True, w_init_gain='linear'):
+        super(LinearNorm, self).__init__()
+        self.linear_layer = torch.nn.Linear(in_dim, out_dim, bias=bias)
+
+        torch.nn.init.xavier_uniform_(
+            self.linear_layer.weight,
+            gain=torch.nn.init.calculate_gain(w_init_gain))
+
+    def forward(self, x):
+        return self.linear_layer(x)
+
+class ResBlk1d(nn.Module):
+    def __init__(self, dim_in, dim_out, actv=nn.LeakyReLU(0.2),
+                 normalize=False, downsample='none', dropout_p=0.2):
+        super().__init__()
+        self.actv = actv
+        self.normalize = normalize
+        self.downsample_type = downsample
+        self.learned_sc = dim_in != dim_out
+        self._build_weights(dim_in, dim_out)
+        self.dropout_p = dropout_p
+        
+        if self.downsample_type == 'none':
+            self.pool = nn.Identity()
+        else:
+            self.pool = weight_norm(nn.Conv1d(dim_in, dim_in, kernel_size=3, stride=2, groups=dim_in, padding=1))
+
+    def _build_weights(self, dim_in, dim_out):
+        self.conv1 = weight_norm(nn.Conv1d(dim_in, dim_in, 3, 1, 1))
+        self.conv2 = weight_norm(nn.Conv1d(dim_in, dim_out, 3, 1, 1))
+        if self.normalize:
+            self.norm1 = nn.InstanceNorm1d(dim_in, affine=True)
+            self.norm2 = nn.InstanceNorm1d(dim_in, affine=True)
+        if self.learned_sc:
+            self.conv1x1 = weight_norm(nn.Conv1d(dim_in, dim_out, 1, 1, 0, bias=False))
+
+    def downsample(self, x):
+        if self.downsample_type == 'none':
+            return x
+        else:
+            if x.shape[-1] % 2 != 0:
+                x = torch.cat([x, x[..., -1].unsqueeze(-1)], dim=-1)
+            return F.avg_pool1d(x, 2)
+
+    def _shortcut(self, x):
+        if self.learned_sc:
+            x = self.conv1x1(x)
+        x = self.downsample(x)
+        return x
+
+    def _residual(self, x):
+        if self.normalize:
+            x = self.norm1(x)
+        x = self.actv(x)
+        x = F.dropout(x, p=self.dropout_p, training=self.training)
+        
+        x = self.conv1(x)
+        x = self.pool(x)
+        if self.normalize:
+            x = self.norm2(x)
+            
+        x = self.actv(x)
+        x = F.dropout(x, p=self.dropout_p, training=self.training)
+        
+        x = self.conv2(x)
+        return x
+
+    def forward(self, x):
+        x = self._shortcut(x) + self._residual(x)
+        return x / math.sqrt(2)  # unit variance
+
+class LayerNorm(nn.Module):
+    def __init__(self, channels, eps=1e-5):
+        super().__init__()
+        self.channels = channels
+        self.eps = eps
+
+        self.gamma = nn.Parameter(torch.ones(channels))
+        self.beta = nn.Parameter(torch.zeros(channels))
+
+    def forward(self, x):
+        x = x.transpose(1, -1)
+        x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
+        return x.transpose(1, -1)
+    
+class TextEncoder(nn.Module):
+    def __init__(self, channels, kernel_size, depth, n_symbols, actv=nn.LeakyReLU(0.2)):
+        super().__init__()
+        self.embedding = nn.Embedding(n_symbols, channels)
+
+        padding = (kernel_size - 1) // 2
+        self.cnn = nn.ModuleList()
+        for _ in range(depth):
+            self.cnn.append(nn.Sequential(
+                weight_norm(nn.Conv1d(channels, channels, kernel_size=kernel_size, padding=padding)),
+                LayerNorm(channels),
+                actv,
+                nn.Dropout(0.2),
+            ))
+        # self.cnn = nn.Sequential(*self.cnn)
+
+        self.lstm = nn.LSTM(channels, channels//2, 1, batch_first=True, bidirectional=True)
+
+    def forward(self, x, input_lengths, m):
+        x = self.embedding(x)  # [B, T, emb]
+        x = x.transpose(1, 2)  # [B, emb, T]
+        m = m.to(input_lengths.device).unsqueeze(1)
+        x.masked_fill_(m, 0.0)
+        
+        for c in self.cnn:
+            x = c(x)
+            x.masked_fill_(m, 0.0)
+            
+        x = x.transpose(1, 2)  # [B, T, chn]
+
+        input_lengths = input_lengths.cpu().numpy()
+        x = nn.utils.rnn.pack_padded_sequence(
+            x, input_lengths, batch_first=True, enforce_sorted=False)
+
+        self.lstm.flatten_parameters()
+        x, _ = self.lstm(x)
+        x, _ = nn.utils.rnn.pad_packed_sequence(
+            x, batch_first=True)
+                
+        x = x.transpose(-1, -2)
+        x_pad = torch.zeros([x.shape[0], x.shape[1], m.shape[-1]])
+
+        x_pad[:, :, :x.shape[-1]] = x
+        x = x_pad.to(x.device)
+        
+        x.masked_fill_(m, 0.0)
+        
+        return x
+
+    def inference(self, x):
+        x = self.embedding(x)
+        x = x.transpose(1, 2)
+        x = self.cnn(x)
+        x = x.transpose(1, 2)
+        self.lstm.flatten_parameters()
+        x, _ = self.lstm(x)
+        return x
+    
+    def length_to_mask(self, lengths):
+        mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths)
+        mask = torch.gt(mask+1, lengths.unsqueeze(1))
+        return mask
+
+
+
+class AdaIN1d(nn.Module):
+    def __init__(self, style_dim, num_features):
+        super().__init__()
+        self.norm = nn.InstanceNorm1d(num_features, affine=False)
+        self.fc = nn.Linear(style_dim, num_features*2)
+
+    def forward(self, x, s):
+        h = self.fc(s)
+        h = h.view(h.size(0), h.size(1), 1)
+        gamma, beta = torch.chunk(h, chunks=2, dim=1)
+        # affine (1 + lin(x)) * inst(x) + lin(x)    is this a skip connection where the weight is a lin of itself
+        return (1 + gamma) * self.norm(x) + beta
+
+class UpSample1d(nn.Module):
+    def __init__(self, layer_type):
+        super().__init__()
+        self.layer_type = layer_type
+
+    def forward(self, x):
+        if self.layer_type == 'none':
+            return x
+        else:
+            return F.interpolate(x, scale_factor=2, mode='nearest')
+
+class AdainResBlk1d(nn.Module):
+    def __init__(self, dim_in, dim_out, style_dim=64, actv=nn.LeakyReLU(0.2),
+                 upsample='none', dropout_p=0.0):
+        super().__init__()
+        self.actv = actv
+        self.upsample_type = upsample
+        self.upsample = UpSample1d(upsample)
+        self.learned_sc = dim_in != dim_out
+        self._build_weights(dim_in, dim_out, style_dim)
+        self.dropout = nn.Dropout(dropout_p)
+        
+        if upsample == 'none':
+            self.pool = nn.Identity()
+        else:
+            self.pool = weight_norm(nn.ConvTranspose1d(dim_in, dim_in, kernel_size=3, stride=2, groups=dim_in, padding=1, output_padding=1))
+        
+        
+    def _build_weights(self, dim_in, dim_out, style_dim):
+        self.conv1 = weight_norm(nn.Conv1d(dim_in, dim_out, 3, 1, 1))
+        self.conv2 = weight_norm(nn.Conv1d(dim_out, dim_out, 3, 1, 1))
+        self.norm1 = AdaIN1d(style_dim, dim_in)
+        self.norm2 = AdaIN1d(style_dim, dim_out)
+        if self.learned_sc:
+            self.conv1x1 = weight_norm(nn.Conv1d(dim_in, dim_out, 1, 1, 0, bias=False))
+
+    def _shortcut(self, x):
+        x = self.upsample(x)
+        if self.learned_sc:
+            x = self.conv1x1(x)
+        return x
+
+    def _residual(self, x, s):
+        x = self.norm1(x, s)
+        x = self.actv(x)
+        x = self.pool(x)
+        x = self.conv1(self.dropout(x))
+        x = self.norm2(x, s)
+        x = self.actv(x)
+        x = self.conv2(self.dropout(x))
+        return x
+
+    def forward(self, x, s):
+        out = self._residual(x, s)
+        out = (out + self._shortcut(x)) / math.sqrt(2)
+        return out
+    
+class AdaLayerNorm(nn.Module):
+    def __init__(self, style_dim, channels, eps=1e-5):
+        super().__init__()
+        self.channels = channels
+        self.eps = eps
+
+        self.fc = nn.Linear(style_dim, channels*2)
+
+    def forward(self, x, s):
+        x = x.transpose(-1, -2)
+        x = x.transpose(1, -1)
+                
+        h = self.fc(s)
+        h = h.view(h.size(0), h.size(1), 1)
+        gamma, beta = torch.chunk(h, chunks=2, dim=1)
+        gamma, beta = gamma.transpose(1, -1), beta.transpose(1, -1)
+        
+        
+        x = F.layer_norm(x, (self.channels,), eps=self.eps)
+        x = (1 + gamma) * x + beta
+        return x.transpose(1, -1).transpose(-1, -2)
+
+class ProsodyPredictor(nn.Module):
+
+    def __init__(self, style_dim, d_hid, nlayers, max_dur=50, dropout=0.1):
+        super().__init__() 
+        
+        self.text_encoder = DurationEncoder(sty_dim=style_dim, 
+                                            d_model=d_hid,
+                                            nlayers=nlayers, 
+                                            dropout=dropout)
+
+        self.lstm = nn.LSTM(d_hid + style_dim, d_hid // 2, 1, batch_first=True, bidirectional=True)
+        self.duration_proj = LinearNorm(d_hid, max_dur)
+        
+        self.shared = nn.LSTM(d_hid + style_dim, d_hid // 2, 1, batch_first=True, bidirectional=True)
+        self.F0 = nn.ModuleList()
+        self.F0.append(AdainResBlk1d(d_hid, d_hid, style_dim, dropout_p=dropout))
+        self.F0.append(AdainResBlk1d(d_hid, d_hid // 2, style_dim, upsample=True, dropout_p=dropout))
+        self.F0.append(AdainResBlk1d(d_hid // 2, d_hid // 2, style_dim, dropout_p=dropout))
+
+        self.N = nn.ModuleList()
+        self.N.append(AdainResBlk1d(d_hid, d_hid, style_dim, dropout_p=dropout))
+        self.N.append(AdainResBlk1d(d_hid, d_hid // 2, style_dim, upsample=True, dropout_p=dropout))
+        self.N.append(AdainResBlk1d(d_hid // 2, d_hid // 2, style_dim, dropout_p=dropout))
+        
+        self.F0_proj = nn.Conv1d(d_hid // 2, 1, 1, 1, 0)
+        self.N_proj = nn.Conv1d(d_hid // 2, 1, 1, 1, 0)
+    
+    def F0Ntrain(self, x, s):
+        x, _ = self.shared(x.transpose(-1, -2))
+        
+        F0 = x.transpose(-1, -2)
+        for block in self.F0:
+            F0 = block(F0, s)
+        F0 = self.F0_proj(F0)
+
+        N = x.transpose(-1, -2)
+        for block in self.N:
+            N = block(N, s)
+        N = self.N_proj(N)
+        
+        return F0.squeeze(1), N.squeeze(1)
+    
+    def length_to_mask(self, lengths):
+        mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths)
+        mask = torch.gt(mask+1, lengths.unsqueeze(1))
+        return mask
+    
+class DurationEncoder(nn.Module):
+
+    def __init__(self, sty_dim, d_model, nlayers, dropout=0.1):
+        super().__init__()
+        self.lstms = nn.ModuleList()
+        for _ in range(nlayers):
+            self.lstms.append(nn.LSTM(d_model + sty_dim, 
+                                 d_model // 2, 
+                                 num_layers=1, 
+                                 batch_first=True, 
+                                 bidirectional=True, 
+                                 dropout=dropout))
+            self.lstms.append(AdaLayerNorm(sty_dim, d_model))
+        
+        
+        self.dropout = dropout
+        self.d_model = d_model
+        self.sty_dim = sty_dim
+
+    def forward(self, x, style, text_lengths, m):
+        masks = m.to(text_lengths.device)
+        
+        x = x.permute(2, 0, 1)
+        s = style.expand(x.shape[0], x.shape[1], -1)
+        x = torch.cat([x, s], axis=-1)
+        x.masked_fill_(masks.unsqueeze(-1).transpose(0, 1), 0.0)
+                
+        x = x.transpose(0, 1)
+        input_lengths = text_lengths.cpu().numpy()
+        x = x.transpose(-1, -2)
+        
+        for block in self.lstms:
+            if isinstance(block, AdaLayerNorm):
+                x = block(x.transpose(-1, -2), style).transpose(-1, -2)
+                x = torch.cat([x, s.permute(1, -1, 0)], axis=1)
+                x.masked_fill_(masks.unsqueeze(-1).transpose(-1, -2), 0.0)
+            else:
+                x = x.transpose(-1, -2)
+                x = nn.utils.rnn.pack_padded_sequence(
+                    x, input_lengths, batch_first=True, enforce_sorted=False)
+                block.flatten_parameters()
+                x, _ = block(x)
+                x, _ = nn.utils.rnn.pad_packed_sequence(
+                    x, batch_first=True)
+                x = F.dropout(x, p=self.dropout, training=self.training)
+                x = x.transpose(-1, -2)
+                
+                x_pad = torch.zeros([x.shape[0], x.shape[1], m.shape[-1]])
+
+                x_pad[:, :, :x.shape[-1]] = x
+                x = x_pad.to(x.device)
+#         print('Calling Duration Encoder\n\n\n\n',x.shape, x.min(), x.max())
+#         Calling Duration Encoder
+#  torch.Size([1, 640, 107]) tensor(-3.0903, device='cuda:0') tensor(2.3089, device='cuda:0')
+        return x.transpose(-1, -2)
+
+    
+    
+    
+def load_F0_models(path):
+    # load F0 model
+
+    F0_model = JDCNet(num_class=1, seq_len=192)
+    print(path, 'WHAT ARE YOU TRYING TO LOAD F0 L520')
+    path.replace('.t7', '.pth')
+    params = torch.load(path, map_location='cpu')['net']
+    F0_model.load_state_dict(params)
+    _ = F0_model.train()
+    
+    return F0_model
+
+def load_ASR_models(ASR_MODEL_PATH, ASR_MODEL_CONFIG):
+    # load ASR model
+    def _load_config(path):
+        with open(path) as f:
+            config = yaml.safe_load(f)
+        model_config = config['model_params']
+        return model_config
+
+    def _load_model(model_config, model_path):
+        model = ASRCNN(**model_config)
+        params = torch.load(model_path, map_location='cpu')['model']
+        model.load_state_dict(params)
+        return model
+
+    asr_model_config = _load_config(ASR_MODEL_CONFIG)
+    asr_model = _load_model(asr_model_config, ASR_MODEL_PATH)
+    _ = asr_model.train()
+
+    return asr_model
+
+def build_model(args, text_aligner, pitch_extractor, bert):
+    print(f'\n==============\n {args.decoder.type=}\n==============L584 models.py @ build_model()\n')
+    
+    from Modules.hifigan import Decoder
+    decoder = Decoder(dim_in=args.hidden_dim, style_dim=args.style_dim, dim_out=args.n_mels,
+            resblock_kernel_sizes = args.decoder.resblock_kernel_sizes,
+            upsample_rates = args.decoder.upsample_rates,
+            upsample_initial_channel=args.decoder.upsample_initial_channel,
+            resblock_dilation_sizes=args.decoder.resblock_dilation_sizes,
+            upsample_kernel_sizes=args.decoder.upsample_kernel_sizes) 
+        
+    text_encoder = TextEncoder(channels=args.hidden_dim, kernel_size=5, depth=args.n_layer, n_symbols=args.n_token)
+    
+    predictor = ProsodyPredictor(style_dim=args.style_dim, d_hid=args.hidden_dim, nlayers=args.n_layer, max_dur=args.max_dur, dropout=args.dropout)
+    
+    style_encoder = StyleEncoder(dim_in=args.dim_in, style_dim=args.style_dim, max_conv_dim=args.hidden_dim) # acoustic style encoder
+    predictor_encoder = StyleEncoder(dim_in=args.dim_in, style_dim=args.style_dim, max_conv_dim=args.hidden_dim) # prosodic style encoder
+        
+    # define diffusion model
+    if args.multispeaker:
+        transformer = StyleTransformer1d(channels=args.style_dim*2, 
+                                    context_embedding_features=bert.config.hidden_size,
+                                    context_features=args.style_dim*2, 
+                                    **args.diffusion.transformer)
+    else:
+        transformer = Transformer1d(channels=args.style_dim*2, 
+                                    context_embedding_features=bert.config.hidden_size,
+                                    **args.diffusion.transformer)
+    
+    diffusion = AudioDiffusionConditional(
+        in_channels=1,
+        embedding_max_length=bert.config.max_position_embeddings,
+        embedding_features=bert.config.hidden_size,
+        embedding_mask_proba=args.diffusion.embedding_mask_proba, # Conditional dropout of batch elements,
+        channels=args.style_dim*2,
+        context_features=args.style_dim*2,
+    )
+    
+    diffusion.diffusion = KDiffusion(
+        net=diffusion.unet,
+        sigma_distribution=LogNormalDistribution(mean = args.diffusion.dist.mean, std = args.diffusion.dist.std),
+        sigma_data=args.diffusion.dist.sigma_data, # a placeholder, will be changed dynamically when start training diffusion model
+        dynamic_threshold=0.0 
+    )
+    diffusion.diffusion.net = transformer
+    diffusion.unet = transformer
+
+    
+    nets = Munch(
+            bert=bert,
+            bert_encoder=nn.Linear(bert.config.hidden_size, args.hidden_dim),
+
+            predictor=predictor,
+            decoder=decoder,
+            text_encoder=text_encoder,
+
+            predictor_encoder=predictor_encoder,
+            style_encoder=style_encoder,
+            diffusion=diffusion,
+
+            text_aligner = text_aligner,
+            pitch_extractor=pitch_extractor
+       )
+    
+    return nets

text_utils.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+import numpy as np
+import re
+import codecs
+# IPA Phonemizer: https://github.com/bootphon/phonemizer
+
+_pad = "$"
+_punctuation = ';:,.!?¡¿—…"«»“” '
+_letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
+_letters_ipa = "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ"
+
+# Export all symbols:
+symbols = [_pad] + list(_punctuation) + list(_letters) + list(_letters_ipa)
+
+dicts = {}
+for i in range(len((symbols))):
+    dicts[symbols[i]] = i
+
+class TextCleaner:
+    def __init__(self, dummy=None):
+        self.word_index_dictionary = dicts
+        print(len(dicts))
+    def __call__(self, text):
+        indexes = []
+        for char in text:
+            try:
+                indexes.append(self.word_index_dictionary[char])
+            except KeyError:
+                print(text)
+        return indexes
+
+
+# == Sentence Splitter
+
+alphabets= "([A-Za-z])"
+prefixes = "(Mr|St|Mrs|Ms|Dr)[.]"
+suffixes = "(Inc|Ltd|Jr|Sr|Co)"
+starters = "(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)"
+acronyms = "([A-Z][.][A-Z][.](?:[A-Z][.])?)"
+websites = "[.](com|net|org|io|gov|edu|me)"
+digits = "([0-9])"
+multiple_dots = r'\.{2,}'
+
+
+
+def split_into_sentences(text):
+    """
+    Split the text into sentences.
+
+    If the text contains substrings "<prd>" or "<stop>", they would lead
+    to incorrect splitting because they are used as markers for splitting.
+
+    :param text: text to be split into sentences
+    :type text: str
+
+    :return: list of sentences
+    :rtype: list[str]
+
+    https://stackoverflow.com/questions/4576077/how-can-i-split-a-text-into-sentences
+    """
+    text = " " + text + "  "
+    text = text.replace("\n"," ")
+    text = re.sub(prefixes,"\\1<prd>",text)
+    text = re.sub(websites,"<prd>\\1",text)
+    text = re.sub(digits + "[.]" + digits,"\\1<prd>\\2",text)
+    text = re.sub(multiple_dots, lambda match: "<prd>" * len(match.group(0)) + "<stop>", text)
+    if "Ph.D" in text: text = text.replace("Ph.D.","Ph<prd>D<prd>")
+    text = re.sub("\s" + alphabets + "[.] "," \\1<prd> ",text)
+    text = re.sub(acronyms+" "+starters,"\\1<stop> \\2",text)
+    text = re.sub(alphabets + "[.]" + alphabets + "[.]" + alphabets + "[.]","\\1<prd>\\2<prd>\\3<prd>",text)
+    text = re.sub(alphabets + "[.]" + alphabets + "[.]","\\1<prd>\\2<prd>",text)
+    text = re.sub(" "+suffixes+"[.] "+starters," \\1<stop> \\2",text)
+    text = re.sub(" "+suffixes+"[.]"," \\1<prd>",text)
+    text = re.sub(" " + alphabets + "[.]"," \\1<prd>",text)
+    if "”" in text: text = text.replace(".”","”.")
+    if "\"" in text: text = text.replace(".\"","\".")
+    if "!" in text: text = text.replace("!\"","\"!")
+    if "?" in text: text = text.replace("?\"","\"?")
+    text = text.replace(".",".<stop>")
+    text = text.replace("?","?<stop>")
+    text = text.replace("!","!<stop>")
+    text = text.replace("<prd>",".")
+    sentences = text.split("<stop>")
+    sentences = [s.strip() for s in sentences]
+    if sentences and not sentences[-1]: sentences = sentences[:-1]
+    return sentences
+
+def store_ssml(text=None,
+               voice=None):
+    '''create ssml:
+           text : list of sentences
+           voice: https://github.com/MycroftAI/mimic3-voices
+    '''
+    print('\n___________________________\n', len(text), text[0], '\n___________________________________\n')
+    _s = '<speak>'
+    for short_text in text:
+
+        rate = min(max(.87, len(short_text) / 76), 1.14) #1.44)  # 1.24 for bieber
+        
+        
+        volume = int(74 * np.random.rand() + 24)
+        # text = ('<speak>'
+        _s += f'<prosody volume=\'{volume}\'>'   # THe other voice does not have volume
+        _s += f'<prosody rate=\'{rate}\'>'
+        _s += f'<voice name=\'{voice}\'>'
+        _s += '<s>'
+        _s += short_text
+        _s += '</s>'
+        _s += '</voice>'
+        _s += '</prosody>'
+        _s += '</prosody>'
+    _s += '</speak>'
+    print(len(text),'\n\n\n\n\n\n\n', _s)
+    
+    with codecs.open('_tmp_ssml.txt', 'w', "utf-8-sig") as f:
+        f.write(_s)

utils.py

@@ -0,0 +1,74 @@
+from monotonic_align import maximum_path
+from monotonic_align import mask_from_lens
+from monotonic_align.core import maximum_path_c
+import numpy as np
+import torch
+import copy
+from torch import nn
+import torch.nn.functional as F
+import torchaudio
+import librosa
+import matplotlib.pyplot as plt
+from munch import Munch
+
+def maximum_path(neg_cent, mask):
+  """ Cython optimized version.
+  neg_cent: [b, t_t, t_s]
+  mask: [b, t_t, t_s]
+  """
+  device = neg_cent.device
+  dtype = neg_cent.dtype
+  neg_cent =  np.ascontiguousarray(neg_cent.data.cpu().numpy().astype(np.float32))
+  path =  np.ascontiguousarray(np.zeros(neg_cent.shape, dtype=np.int32))
+
+  t_t_max = np.ascontiguousarray(mask.sum(1)[:, 0].data.cpu().numpy().astype(np.int32))
+  t_s_max = np.ascontiguousarray(mask.sum(2)[:, 0].data.cpu().numpy().astype(np.int32))
+  maximum_path_c(path, neg_cent, t_t_max, t_s_max)
+  return torch.from_numpy(path).to(device=device, dtype=dtype)
+
+def get_data_path_list(train_path=None, val_path=None):
+    if train_path is None:
+        train_path = "Data/train_list.txt"
+    if val_path is None:
+        val_path = "Data/val_list.txt"
+
+    with open(train_path, 'r', encoding='utf-8', errors='ignore') as f:
+        train_list = f.readlines()
+    with open(val_path, 'r', encoding='utf-8', errors='ignore') as f:
+        val_list = f.readlines()
+
+    return train_list, val_list
+
+def length_to_mask(lengths):
+    mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths)
+    mask = torch.gt(mask+1, lengths.unsqueeze(1))
+    return mask
+
+# for norm consistency loss
+def log_norm(x, mean=-4, std=4, dim=2):
+    """
+    normalized log mel -> mel -> norm -> log(norm)
+    """
+    x = torch.log(torch.exp(x * std + mean).norm(dim=dim))
+    return x
+
+def get_image(arrs):
+    plt.switch_backend('agg')
+    fig = plt.figure()
+    ax = plt.gca()
+    ax.imshow(arrs)
+
+    return fig
+
+def recursive_munch(d):
+    if isinstance(d, dict):
+        return Munch((k, recursive_munch(v)) for k, v in d.items())
+    elif isinstance(d, list):
+        return [recursive_munch(v) for v in d]
+    else:
+        return d
+    
+def log_print(message, logger):
+    logger.info(message)
+    print(message)
+