content: /**
   * Copyright 2007 DFKI GmbH.
   * All Rights Reserved.  Use is subject to license terms.
   * <p/>
   * This file is part of MARY TTS.
   * <p/>
   * MARY TTS is free software: you can redistribute it and/or modify
   * it under the terms of the GNU Lesser General Public License as published by
   * the Free Software Foundation, version 3 of the License.
   * <p/>
   * This program is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   * GNU Lesser General Public License for more details.
   * <p/>
   * You should have received a copy of the GNU Lesser General Public License
   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
   */
  package marytts.signalproc.sinusoidal;
  
  import android.util.Log;
  
  import java.io.File;
  
  import lib.sound.sampled.AudioFileFormat;
  import lib.sound.sampled.AudioInputStream;
  import lib.sound.sampled.AudioSystem;
  import marytts.server.Mary;
  import marytts.signalproc.analysis.PitchMarks;
  import marytts.signalproc.analysis.PitchReaderWriter;
  import marytts.signalproc.filter.FilterBankAnalyserBase;
  import marytts.signalproc.window.Window;
  import marytts.util.data.BufferedDoubleDataSource;
  import marytts.util.data.audio.AudioDoubleDataSource;
  import marytts.util.data.audio.DDSAudioInputStream;
  import marytts.util.math.MathUtils;
  import marytts.util.signal.SignalProcUtils;
  
  /**
   * Prosody modification using sinusoidal model.
   * <p/>
   * Reference:
   * Quatieri, T. F. and R. J. McAula y, 1992, “Shape Invariant Timescale and Pitch Modification of
   * Speech”, IEEE Transactions On Signal Processing, vol. 40, no. 3, pp. 497-510.
   *
   * @author Oytun T&uumlrk
   */
  public class ProsodyModifier {
      public int fs;
  
      public ProsodyModifier(int samplingRate) {
          fs = samplingRate;
      }
  
      public static void main(String[] args) throws Exception {
          if (false) {
              //File input
              AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(args[0]));
              int samplingRate = (int) inputAudio.getFormat().getSampleRate();
              AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio);
              double[] x = signal.getAllData();
  
              //Read pitch contour (real speech or create it from pm file
              PitchReaderWriter f0 = null;
              String strPitchFile = args[0].substring(0, args[0].length() - 4) + ".ptc";
              f0 = new PitchReaderWriter(strPitchFile);
              //
  
              //Analysis
              float deltaInHz = SinusoidalAnalysisParams.DEFAULT_DELTA_IN_HZ;
              float numPeriods = PitchSynchronousSinusoidalAnalyzer.DEFAULT_ANALYSIS_PERIODS;
              boolean isSilentSynthesis = false;
  
              boolean bRefinePeakEstimatesParabola = false;
              boolean bRefinePeakEstimatesBias = false;
              boolean bSpectralReassignment = false;
              boolean bAdjustNeighFreqDependent = false;
  
              double absMaxOriginal = MathUtils.getAbsMax(x);
  
              float skipSizeInSeconds = TrackModifier.DEFAULT_MODIFICATION_SKIP_SIZE;
              //skipSizeInSeconds = -1.0f;
              //skipSizeInSeconds = 0.002f;
  
              boolean isVoicingAdaptiveTimeScaling = true;
              float timeScalingVoicingThreshold = 0.5f;
              boolean isVoicingAdaptivePitchScaling = true;
  
              int spectralEnvelopeType = SinusoidalAnalysisParams.LP_SPEC;
  
              //int analyzerType = BaseSinusoidalAnalyzer.FIXEDRATE_FULLBAND_ANALYZER;
              //int analyzerType = BaseSinusoidalAnalyzer.PITCHSYNCHRONOUS_FULLBAND_ANALYZER;
              int pitchMarkOffset = 0; //Only used in PITCHSYNCHRONOUS_FULLBAND_ANALYZER
              int analyzerType = BaseSinusoidalAnalyzer.FIXEDRATE_MULTIRESOLUTION_ANALYZER;
  
              int synthesizerType = BaseSinusoidalSynthesizer.PEAK_MATCHED_TRACK_SYNTHESIZER;
              //int synthesizerType = BaseSinusoidalSynthesizer.OVERLAP_ADD_SYNTHESIZER;
  
              ProsodyModifier pm = new ProsodyModifier(samplingRate);
              double[] y = null;
  
              int sysPhaseModMethod, sysAmpModMethod;
  
              sysPhaseModMethod = TrackModifier.FROM_ORIGINAL;
              //sysPhaseModMethod = TrackModifier.FROM_RESAMPLED;
              //sysPhaseModMethod = TrackModifier.FROM_CEPSTRUM;
              //sysPhaseModMethod = TrackModifier.FROM_INTERPOLATED; //This is only available for phase
  
              sysAmpModMethod = TrackModifier.FROM_ORIGINAL;
              //sysAmpModMethod = TrackModifier.FROM_RESAMPLED;
              //sysAmpModMethod = TrackModifier.FROM_CEPSTRUM;
  
              if (true) {
                  float timeScale = 1.0f;
                  float pitchScale = 1.05f;
                  y = pm.process(x, f0.contour, (float) f0.header.windowSizeInSeconds, (float) f0.header.skipSizeInSeconds,
                          isVoicingAdaptiveTimeScaling, timeScalingVoicingThreshold, isVoicingAdaptivePitchScaling,
                          timeScale, pitchScale, skipSizeInSeconds, deltaInHz, numPeriods,
                          bRefinePeakEstimatesParabola, bRefinePeakEstimatesBias, bSpectralReassignment, bAdjustNeighFreqDependent, isSilentSynthesis,
                          absMaxOriginal, spectralEnvelopeType, analyzerType, synthesizerType, pitchMarkOffset, sysPhaseModMethod, sysAmpModMethod);
              } else {
                  float[] timeScales = {0.5f, 0.75f, 1.25f, 1.75f};
                  float[] timeScalesTimes = {0.5f, 1.25f, 2.0f, 2.5f};
                  float[] pitchScales = {2.0f, 1.5f, 0.8f, 0.6f};
                  float[] pitchScalesTimes = {0.5f, 1.25f, 2.0f, 2.5f};
  
                  y = pm.process(x, f0.contour, (float) f0.header.windowSizeInSeconds, (float) f0.header.skipSizeInSeconds,
                          isVoicingAdaptiveTimeScaling, timeScalingVoicingThreshold, isVoicingAdaptivePitchScaling,
                          timeScales, timeScalesTimes, pitchScales, pitchScalesTimes, skipSizeInSeconds, deltaInHz, numPeriods,
                          bRefinePeakEstimatesParabola, bRefinePeakEstimatesBias, bSpectralReassignment, bAdjustNeighFreqDependent, isSilentSynthesis,
                          absMaxOriginal, spectralEnvelopeType, analyzerType, synthesizerType, pitchMarkOffset, sysPhaseModMethod, sysAmpModMethod);
              }
  
              //File output
              DDSAudioInputStream outputAudio = new DDSAudioInputStream(new BufferedDoubleDataSource(y), inputAudio.getFormat());
              String outFileName = args[0].substring(0, args[0].length() - 4) + "_sinProsodyModified.wav";
              AudioSystem.write(outputAudio, AudioFileFormat.Type.WAVE, new File(outFileName));
              //
          } else
              main2(args);
      }
  
      public static void main2(String[] args) throws Exception {
          //File input
          AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(args[0]));
          int samplingRate = (int) inputAudio.getFormat().getSampleRate();
          AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio);
          double[] x = signal.getAllData();
  
  
          //Read pitch contour (real speech or create it from pm file
          PitchReaderWriter f0 = null;
          String strPitchFile = args[0].substring(0, args[0].length() - 4) + ".ptc";
          f0 = new PitchReaderWriter(strPitchFile);
          //
  
          //Analysis
          float deltaInHz = SinusoidalAnalysisParams.DEFAULT_DELTA_IN_HZ;
          float numPeriods = PitchSynchronousSinusoidalAnalyzer.DEFAULT_ANALYSIS_PERIODS;
          boolean isSilentSynthesis = false;
  
          boolean bRefinePeakEstimatesParabola = false;
          boolean bRefinePeakEstimatesBias = false;
          boolean bSpectralReassignment = false;
          boolean bAdjustNeighFreqDependent = false;
  
          double absMaxOriginal = MathUtils.getAbsMax(x);
  
          float skipSizeInSeconds = TrackModifier.DEFAULT_MODIFICATION_SKIP_SIZE;
          //skipSizeInSeconds = -1.0f;
          //skipSizeInSeconds = 0.002f;
  
          boolean isVoicingAdaptiveTimeScaling = false;
          float timeScalingVoicingThreshold = 0.5f;
          boolean isVoicingAdaptivePitchScaling = false;
  
          //int spectralEnvelopeType = SinusoidalAnalysisParams.LP_SPEC; String envelopeName="lp";
          int spectralEnvelopeType = SinusoidalAnalysisParams.SEEVOC_SPEC;
          String envelopeName = "sv";
          //int spectralEnvelopeType = SinusoidalAnalysisParams.REGULARIZED_CEPS; String envelopeName="rc";
  
          //int analyzerType = BaseSinusoidalAnalyzer.FIXEDRATE_FULLBAND_ANALYZER;
          int analyzerType = BaseSinusoidalAnalyzer.PITCHSYNCHRONOUS_FULLBAND_ANALYZER;
          int pitchMarkOffset = 0; //Only used in PITCHSYNCHRONOUS_FULLBAND_ANALYZER
          //int analyzerType = BaseSinusoidalAnalyzer.FIXEDRATE_MULTIRESOLUTION_ANALYZER;
  
          int synthesizerType = BaseSinusoidalSynthesizer.PEAK_MATCHED_TRACK_SYNTHESIZER;
          //int synthesizerType = BaseSinusoidalSynthesizer.OVERLAP_ADD_SYNTHESIZER;
  
          ProsodyModifier pm = new ProsodyModifier(samplingRate);
          double[] y = null;
  
          float pitchScale;
          float timeScale;
          DDSAudioInputStream outputAudio;
          String outFileName;
  
          //String[] pitchScales = {"0.80", "0.90", "0.95", "0.99", "1.01", "1.05", "1.10", "1.20"};
          //String[] pitchScales = {"2.0"};
          String pitchMarkOffsetStr;
  
          int sysPhaseModMethod, sysAmpModMethod;
  
          //sysPhaseModMethod = TrackModifier.FROM_ORIGINAL;
          //sysPhaseModMethod = TrackModifier.FROM_RESAMPLED; //Best so far
          sysPhaseModMethod = TrackModifier.FROM_CEPSTRUM; //Best so far
          //sysPhaseModMethod = TrackModifier.FROM_INTERPOLATED; //This is only available for phase mod, however it does not work properly, need to check phase paper of Quatieri
  
          //sysAmpModMethod = TrackModifier.FROM_ORIGINAL;
          sysAmpModMethod = TrackModifier.FROM_RESAMPLED; //Best so far
          //sysAmpModMethod = TrackModifier.FROM_CEPSTRUM;
  
          int n = 0;
          //for (n=0; n<=58; n++)
          {
              pitchMarkOffset = n;
  
              pitchMarkOffsetStr = String.valueOf(pitchMarkOffset);
              if (n < 10)
                  pitchMarkOffsetStr = "0" + pitchMarkOffsetStr;
  
              timeScale = 1.0f;
              pitchScale = 4.0f;
              //for (int i=0; i<pitchScales.length; i++)
              {
                  y = pm.process(x, f0.contour, (float) f0.header.windowSizeInSeconds, (float) f0.header.skipSizeInSeconds,
                          isVoicingAdaptiveTimeScaling, timeScalingVoicingThreshold, isVoicingAdaptivePitchScaling,
                          //timeScale, Float.valueOf(pitchScales[i]), skipSizeInSeconds, deltaInHz, numPeriods,
                          timeScale, pitchScale, skipSizeInSeconds, deltaInHz, numPeriods,
                          bRefinePeakEstimatesParabola, bRefinePeakEstimatesBias, bSpectralReassignment, bAdjustNeighFreqDependent, isSilentSynthesis,
                          absMaxOriginal, spectralEnvelopeType, analyzerType, synthesizerType, pitchMarkOffset,
                          sysPhaseModMethod, sysAmpModMethod);
                  outputAudio = new DDSAudioInputStream(new BufferedDoubleDataSource(y), inputAudio.getFormat());
                  outFileName = args[0].substring(0, args[0].length() - 4) + "_sin_t" + String.valueOf(timeScale) + "_p" + String.valueOf(pitchScale) + "_phaseMod" + String.valueOf(sysPhaseModMethod) + "_ampMod" + String.valueOf(sysAmpModMethod) + ".wav";
                  AudioSystem.write(outputAudio, AudioFileFormat.Type.WAVE, new File(outFileName));
              }
          }
  
          System.out.println("Sinusoidal prosody modifications completed...");
      }
  
      public double[] process(double[] x,
                              double[] f0s,
                              float f0_ws, float f0_ss,
                              boolean isVoicingAdaptiveTimeScaling,
                              float timeScalingVoicingThreshold,
                              boolean isVoicingAdaptivePitchScaling,
                              float timeScale,
                              float pitchScale,
                              float skipSizeInSeconds,
                              float deltaInHz,
                              float numPeriods,
                              boolean bRefinePeakEstimatesParabola,
                              boolean bRefinePeakEstimatesBias,
                              boolean bSpectralReassignment,
                              boolean bAdjustNeighFreqDependent,
                              boolean isSilentSynthesis,
                              double absMaxDesired,
                              int spectralEnvelopeType,
                              int analyzerType,
                              int synthesizerType,
                              int pitchMarkOffset,
                              int sysPhaseModMethod,
                              int sysAmpModMethod) throws Exception {
          float[] tScales = new float[1];
          float[] tScalesTimes = new float[1];
          tScales[0] = timeScale;
          tScalesTimes[0] = skipSizeInSeconds;
  
          float[] pScales = new float[1];
          float[] pScalesTimes = new float[1];
          pScales[0] = pitchScale;
          pScalesTimes[0] = skipSizeInSeconds;
  
          return process(x,
                  f0s,
                  f0_ws, f0_ss,
                  isVoicingAdaptiveTimeScaling,
                  timeScalingVoicingThreshold,
                  isVoicingAdaptivePitchScaling,
                  tScales,
                  tScalesTimes,
                  pScales,
                  pScalesTimes,
                  skipSizeInSeconds,
                  deltaInHz,
                  numPeriods,
                  bRefinePeakEstimatesParabola,
                  bRefinePeakEstimatesBias,
                  bSpectralReassignment,
                  bAdjustNeighFreqDependent,
                  isSilentSynthesis,
                  absMaxDesired,
                  spectralEnvelopeType,
                  analyzerType,
                  synthesizerType,
                  pitchMarkOffset,
                  sysPhaseModMethod,
                  sysAmpModMethod);
      }
  
      public double[] process(double[] x,
                              double[] f0s,
                              float f0_ws, float f0_ss,
                              boolean isVoicingAdaptiveTimeScaling,
                              float timeScalingVoicingThreshold,
                              boolean isVoicingAdaptivePitchScaling,
                              float[] timeScales,
                              float[] timeScalesTimes,
                              float[] pitchScales,
                              float[] pitchScalesTimes,
                              float skipSizeInSeconds,
                              float deltaInHz,
                              float numPeriods,
                              boolean bRefinePeakEstimatesParabola,
                              boolean bRefinePeakEstimatesBias,
                              boolean bSpectralReassignment,
                              boolean bAdjustNeighFreqDependent,
                              boolean isSilentSynthesis,
                              double absMaxDesired,
                              int spectralEnvelopeType,
                              int analyzerType,
                              int synthesizerType,
                              int pitchMarkOffset,
                              int sysPhaseModMethod,
                              int sysAmpModMethod) throws Exception {
          int windowType = Window.HANNING;
          //Analysis
          PitchMarks pm = SignalProcUtils.pitchContour2pitchMarks(f0s, fs, x.length, f0_ws, f0_ss, false, pitchMarkOffset);
  
          BaseSinusoidalAnalyzer an = null;
          SinusoidalTracks[] st = null;
          SinusoidalAnalysisParams params = new SinusoidalAnalysisParams(fs, 0.0, 0.5 * fs, windowType,
                  bRefinePeakEstimatesParabola,
                  bRefinePeakEstimatesBias,
                  bSpectralReassignment,
                  bAdjustNeighFreqDependent);
  
          if (analyzerType == BaseSinusoidalAnalyzer.FIXEDRATE_FULLBAND_ANALYZER) {
              an = new SinusoidalAnalyzer(params);
  
              st = new SinusoidalTracks[1];
              st[0] = ((SinusoidalAnalyzer) an).analyzeFixedRate(x, f0_ws, f0_ss, deltaInHz, spectralEnvelopeType, f0s, f0_ws, f0_ss);
          } else if (analyzerType == BaseSinusoidalAnalyzer.PITCHSYNCHRONOUS_FULLBAND_ANALYZER) {
              an = new PitchSynchronousSinusoidalAnalyzer(params);
  
              st = new SinusoidalTracks[1];
              st[0] = ((PitchSynchronousSinusoidalAnalyzer) an).analyzePitchSynchronous(x, pm, numPeriods, skipSizeInSeconds, deltaInHz, spectralEnvelopeType);
          } else if (analyzerType == BaseSinusoidalAnalyzer.FIXEDRATE_MULTIRESOLUTION_ANALYZER) {
              //These should be input as well
              int multiresolutionFilterbankType = FilterBankAnalyserBase.FIR_BANDPASS_FILTERBANK;
              int numBands = 4;
              double lowestBandWindowSizeInSeconds = 0.020;
              boolean bFreqLimitedAnalysis = true;
              //
  
              an = new MultiresolutionSinusoidalAnalyzer(multiresolutionFilterbankType, numBands, fs);
  
  
              st = ((MultiresolutionSinusoidalAnalyzer) an).analyze(x,
                      lowestBandWindowSizeInSeconds,
                      windowType,
                      bRefinePeakEstimatesParabola,
                      bRefinePeakEstimatesBias,
                      bSpectralReassignment,
                      bAdjustNeighFreqDependent,
                      bFreqLimitedAnalysis);
          } else if (analyzerType == BaseSinusoidalAnalyzer.PITCHSYNCHRONOUS_MULTIRESOLUTION_ANALYZER) {
              /*
              an = new MultiresolutionSinusoidalAnalyzer();
  
              //These should be input as well
              int multiresolutionFilterbankType = MultiresolutionSinusoidalAnalyzer.FIR_BANDPASS_FILTERBANK;
              int numBands = 4;
              double lowestBandWindowSizeInSeconds = 0.020;
              //
  
              st = ((MultiresolutionSinusoidalAnalyzer)an).analyzePitchSynchronous(x, fs,
                                                                            multiresolutionFilterbankType,
                                                                            numBands,
                                                                            lowestBandWindowSizeInSeconds,
                                                                            windowType,
                                                                            bRefinePeakEstimatesParabola,
                                                                            bRefinePeakEstimatesBias,
                                                                            bSpectralReassignment,
                                                                            bAdjustNeighFreqDependent);
                                                                            */
          } else {
              System.out.println("Invalid sinusoidal analyzer...");
              return null;
          }
  
          //To do: Estimation of voicing probabilities...
  
          SinusoidalTracks[] stMod = new SinusoidalTracks[st.length];
  
          for (int i = 0; i < st.length; i++) {
              stMod[i] = TrackModifier.modify(st[i], f0s, f0_ss, f0_ws, pm.pitchMarks, st[i].voicings, numPeriods,
                      isVoicingAdaptiveTimeScaling, timeScalingVoicingThreshold,
                      isVoicingAdaptivePitchScaling,
                      timeScales, timeScalesTimes, pitchScales, pitchScalesTimes, pitchMarkOffset,
                      sysAmpModMethod, sysPhaseModMethod);
          }
  
          //Synthesis
          double[] y = null;
          if (synthesizerType == BaseSinusoidalSynthesizer.PEAK_MATCHED_TRACK_SYNTHESIZER) {
              PeakMatchedSinusoidalSynthesizer synthesizer = new PeakMatchedSinusoidalSynthesizer(fs);
              y = synthesizer.synthesize(stMod, isSilentSynthesis);
          } else if (synthesizerType == BaseSinusoidalSynthesizer.PEAK_MATCHED_TRACK_SYNTHESIZER) {
              OverlapAddSinusoidalSynthesizer synthesizer = new OverlapAddSinusoidalSynthesizer(fs);
              y = synthesizer.synthesize(stMod, isSilentSynthesis);
          } else {
              Log.d(Mary.LOG, new Exception("Unknown sinusoidal synthesizer type!").toString());
              throw new Exception("Unknown sinusoidal synthesizer type!");
  
          }
  
          if (y != null) {
              //y = OverlapAddEnergyNormalizer.normalize(y, st[0].fs, f0_ws, f0_ss, windowType, 1.0);
              y = MathUtils.normalizeToAbsMax(y, absMaxDesired);
          }
  
          return y;
      }
  }
  
  