app.py

import sys,os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from vits.models import SynthesizerInfer
from omegaconf import OmegaConf
import torchcrepe
import torch
import io
import os
import gradio as gr
import librosa
import numpy as np
import soundfile

import logging

logging.getLogger('numba').setLevel(logging.WARNING)
logging.getLogger('markdown_it').setLevel(logging.WARNING)
logging.getLogger('urllib3').setLevel(logging.WARNING)
logging.getLogger('matplotlib').setLevel(logging.WARNING)


def load_svc_model(checkpoint_path, model):
    assert os.path.isfile(checkpoint_path)
    checkpoint_dict = torch.load(checkpoint_path, map_location="cpu")
    saved_state_dict = checkpoint_dict["model_g"]
    state_dict = model.state_dict()
    new_state_dict = {}
    for k, v in state_dict.items():
        new_state_dict[k] = saved_state_dict[k]
    model.load_state_dict(new_state_dict)
    return model


def compute_f0_nn(filename, device):
    audio, sr = librosa.load(filename, sr=16000)
    assert sr == 16000
    # Load audio
    audio = torch.tensor(np.copy(audio))[None]
    # Here we'll use a 20 millisecond hop length
    hop_length = 320
    # Provide a sensible frequency range for your domain (upper limit is 2006 Hz)
    # This would be a reasonable range for speech
    fmin = 50
    fmax = 1000
    # Select a model capacity--one of "tiny" or "full"
    model = "full"
    # Pick a batch size that doesn't cause memory errors on your gpu
    batch_size = 512
    # Compute pitch using first gpu
    pitch, periodicity = torchcrepe.predict(
        audio,
        sr,
        hop_length,
        fmin,
        fmax,
        model,
        batch_size=batch_size,
        device=device,
        return_periodicity=True,
    )
    pitch = np.repeat(pitch, 2, -1)  # 320 -> 160 * 2
    periodicity = np.repeat(periodicity, 2, -1)  # 320 -> 160 * 2
    # CREPE was not trained on silent audio. some error on silent need filter.
    periodicity = torchcrepe.filter.median(periodicity, 9)
    pitch = torchcrepe.filter.mean(pitch, 3)
    # pitch[periodicity < 0.1] = 0
    pitch = pitch.squeeze(0)
    return pitch


device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
hp = OmegaConf.load("configs/base.yaml")
model = SynthesizerInfer(
    hp.data.filter_length // 2 + 1,
    hp.data.segment_size // hp.data.hop_length,
    hp)
load_svc_model("vits_pretrain/sovits5.0_bigvgan_mix.pth", model)
model.eval()
model.to(device)


def svc_change(argswave, argsspk):

    argsppg = "svc_tmp.ppg.npy"
    os.system(f"python whisper/inference.py -w {argswave} -p {argsppg}")
    argsvec = "svc_tmp.vec.npy"
    os.system(f"python hubert/inference.py -w {argswave} -v {argsvec}")

    spk = np.load(argsspk)
    spk = torch.FloatTensor(spk)

    ppg = np.load(argsppg)
    ppg = np.repeat(ppg, 2, 0)  # 320 PPG -> 160 * 2
    ppg = torch.FloatTensor(ppg)

    vec = np.load(argsvec)
    vec = np.repeat(vec, 2, 0)  # 320 PPG -> 160 * 2
    vec = torch.FloatTensor(vec)

    pit = compute_f0_nn(argswave, device)
    pit = torch.FloatTensor(pit)

    len_pit = pit.size()[0]
    len_vec = vec.size()[0]
    len_ppg = ppg.size()[0]
    len_min = min(len_pit, len_vec)
    len_min = min(len_min, len_ppg)
    pit = pit[:len_min]
    vec = vec[:len_min, :]
    ppg = ppg[:len_min, :]

    with torch.no_grad():

        spk = spk.unsqueeze(0).to(device)
        source = pit.unsqueeze(0).to(device)
        source = model.pitch2source(source)

        hop_size = hp.data.hop_length
        all_frame = len_min
        hop_frame = 10
        out_chunk = 2500  # 25 S
        out_index = 0
        out_audio = []
        has_audio = False

        while (out_index + out_chunk < all_frame):
            has_audio = True
            if (out_index == 0):  # start frame
                cut_s = out_index
                cut_s_out = 0
            else:
                cut_s = out_index - hop_frame
                cut_s_out = hop_frame * hop_size

            if (out_index + out_chunk + hop_frame > all_frame):  # end frame
                cut_e = out_index + out_chunk
                cut_e_out = 0
            else:
                cut_e = out_index + out_chunk + hop_frame
                cut_e_out = -1 * hop_frame * hop_size

            sub_ppg = ppg[cut_s:cut_e, :].unsqueeze(0).to(device)
            sub_vec = vec[cut_s:cut_e, :].unsqueeze(0).to(device)
            sub_pit = pit[cut_s:cut_e].unsqueeze(0).to(device)
            sub_len = torch.LongTensor([cut_e - cut_s]).to(device)
            sub_har = source[:, :, cut_s *
                             hop_size:cut_e * hop_size].to(device)
            sub_out = model.inference(sub_ppg, sub_vec, sub_pit, spk, sub_len, sub_har)
            sub_out = sub_out[0, 0].data.cpu().detach().numpy()

            sub_out = sub_out[cut_s_out:cut_e_out]
            out_audio.extend(sub_out)
            out_index = out_index + out_chunk

        if (out_index < all_frame):
            if (has_audio):
                cut_s = out_index - hop_frame
                cut_s_out = hop_frame * hop_size
            else:
                cut_s = 0
                cut_s_out = 0
            sub_ppg = ppg[cut_s:, :].unsqueeze(0).to(device)
            sub_vec = vec[cut_s:, :].unsqueeze(0).to(device)
            sub_pit = pit[cut_s:].unsqueeze(0).to(device)
            sub_len = torch.LongTensor([all_frame - cut_s]).to(device)
            sub_har = source[:, :, cut_s * hop_size:].to(device)
            sub_out = model.inference(sub_ppg, sub_vec, sub_pit, spk, sub_len, sub_har)
            sub_out = sub_out[0, 0].data.cpu().detach().numpy()

            sub_out = sub_out[cut_s_out:]
            out_audio.extend(sub_out)
        out_audio = np.asarray(out_audio)

    return out_audio


def svc_main(sid, input_audio):
    if input_audio is None:
        return "You need to upload an audio", None
    sampling_rate, audio = input_audio
    audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
    if len(audio.shape) > 1:
        audio = librosa.to_mono(audio.transpose(1, 0))
    if sampling_rate != 16000:
        audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000)
    if (len(audio) > 16000*100):
        audio = audio[:16000*100]
    wav_path = "temp.wav"
    soundfile.write(wav_path, audio, 16000, format="wav")
    out_audio = svc_change(wav_path, f"configs/singers/singer00{sid}.npy")
    return "Success", (32000, out_audio)


app = gr.Blocks()
with app:
    with gr.Tabs():
        with gr.TabItem("sovits 5.0"):
            gr.Markdown(value="""
                基于开源数据:Multi-Singer
                
                https://github.com/Multi-Singer/Multi-Singer.github.io
                
                最终版本：

                1，mix_encoder: whisper + hubert, 解决跨语言转换和纯对白语音训练
                
                2，解决F0瑕疵

                """)
            sid = gr.Dropdown(label="音色", choices=[
                              "22", "33", "47", "51"], value="47")
            vc_input3 = gr.Audio(label="上传音频")
            vc_submit = gr.Button("转换", variant="primary")
            vc_output1 = gr.Textbox(label="状态信息")
            vc_output2 = gr.Audio(label="转换音频")
        vc_submit.click(svc_main, [sid, vc_input3], [vc_output1, vc_output2])

    app.launch()