init

.gitignore

@@ -0,0 +1,63 @@
+# Mac OS files
+.DS_Store
+ckpt
+# IDEs
+.idea
+.vs
+.vscode
+.cache
+*.png
+# GitHub files
+.github
+
+# Byte-compiled / optimized / DLL / cached files
+__pycache__/
+*.py[cod]
+*$py.class
+*.pyc
+.temp
+*.c
+*.so
+*.o
+
+# Developing mode
+_*.sh
+_*.json
+*.lst
+yard*
+*.out
+evaluation/evalset_selection
+mfa
+egs/svc/*wavmark
+egs/svc/custom
+egs/svc/*/dev*
+egs/svc/dev_exp_config.json
+bins/svc/demo*
+bins/svc/preprocess_custom.py
+data
+ckpts
+
+# Data and ckpt
+*.pkl
+*.pt
+*.npy
+*.npz
+*.tar.gz
+*.ckpt
+*.wav
+*.flac
+pretrained/wenet/*conformer_exp
+!egs/tts/VALLE/prompt_examples/*.wav
+*.bin
+
+# Runtime data dirs
+processed_data
+data
+model_ckpt
+logs
+*.ipynb
+*.lst
+source_audio
+result
+conversion_results
+get_available_gpu.py

Dockerfile

@@ -0,0 +1,64 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Other version: https://hub.docker.com/r/nvidia/cuda/tags
+FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu18.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG PYTORCH='2.0.0'
+ARG CUDA='cu118'
+ARG SHELL='/bin/bash'
+ARG MINICONDA='Miniconda3-py39_23.3.1-0-Linux-x86_64.sh'
+
+ENV LANG=en_US.UTF-8 PYTHONIOENCODING=utf-8 PYTHONDONTWRITEBYTECODE=1 CUDA_HOME=/usr/local/cuda CONDA_HOME=/opt/conda SHELL=${SHELL}
+ENV PATH=$CONDA_HOME/bin:$CUDA_HOME/bin:$PATH \
+    LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH \
+    LIBRARY_PATH=$CUDA_HOME/lib64:$LIBRARY_PATH \
+    CONDA_PREFIX=$CONDA_HOME \
+    NCCL_HOME=$CUDA_HOME
+
+# Install ubuntu packages
+RUN sed -i 's/archive.ubuntu.com/mirrors.cloud.tencent.com/g' /etc/apt/sources.list \
+    && sed -i 's/security.ubuntu.com/mirrors.cloud.tencent.com/g' /etc/apt/sources.list \
+    && rm /etc/apt/sources.list.d/cuda.list \
+    && apt-get update \
+    && apt-get -y install \
+    python3-pip ffmpeg git less wget libsm6 libxext6 libxrender-dev \
+    build-essential cmake pkg-config libx11-dev libatlas-base-dev \
+    libgtk-3-dev libboost-python-dev vim libgl1-mesa-glx \
+    libaio-dev software-properties-common tmux \
+    espeak-ng
+
+# Install miniconda with python 3.9
+USER root
+# COPY Miniconda3-py39_23.3.1-0-Linux-x86_64.sh /root/anaconda.sh
+RUN wget -t 0 -c -O /tmp/anaconda.sh https://repo.anaconda.com/miniconda/${MINICONDA} \
+    && mv /tmp/anaconda.sh /root/anaconda.sh \
+    && ${SHELL} /root/anaconda.sh -b -p $CONDA_HOME \
+    && rm /root/anaconda.sh
+
+RUN conda create -y --name amphion python=3.9.15
+
+WORKDIR /app
+COPY env.sh env.sh
+RUN chmod +x ./env.sh
+
+RUN ["conda", "run", "-n", "amphion", "-vvv", "--no-capture-output", "./env.sh"]
+
+RUN conda init \
+    && echo "\nconda activate amphion\n" >> ~/.bashrc
+
+CMD ["/bin/bash"]
+
+# *** Build ***
+# docker build -t realamphion/amphion .
+
+# *** Run ***
+# cd Amphion
+# docker run --runtime=nvidia --gpus all -it -v .:/app -v /mnt:/mnt_host realamphion/amphion
+
+# *** Push and release ***
+# docker login
+# docker push realamphion/amphion

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Amphion
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,163 @@
+# Amphion: An Open-Source Audio, Music, and Speech Generation Toolkit
+
+<div>
+    <a href="https://arxiv.org/abs/2312.09911"><img src="https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg"></a>
+    <a href="https://huggingface.co/amphion"><img src="https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace-Amphion-pink"></a>
+    <a href="https://openxlab.org.cn/usercenter/Amphion"><img src="https://cdn-static.openxlab.org.cn/app-center/openxlab_app.svg"></a>
+    <a href="egs/tts/README.md"><img src="https://img.shields.io/badge/README-TTS-blue"></a>
+    <a href="egs/svc/README.md"><img src="https://img.shields.io/badge/README-SVC-blue"></a>
+    <a href="egs/tta/README.md"><img src="https://img.shields.io/badge/README-TTA-blue"></a>
+    <a href="egs/vocoder/README.md"><img src="https://img.shields.io/badge/README-Vocoder-purple"></a>
+    <a href="egs/metrics/README.md"><img src="https://img.shields.io/badge/README-Evaluation-yellow"></a>
+    <a href="LICENSE"><img src="https://img.shields.io/badge/LICENSE-MIT-red"></a>
+</div>
+<br>
+
+**Amphion (/æmˈfaɪən/) is a toolkit for Audio, Music, and Speech Generation.** Its purpose is to support reproducible research and help junior researchers and engineers get started in the field of audio, music, and speech generation research and development. Amphion offers a unique feature: **visualizations** of classic models or architectures. We believe that these visualizations are beneficial for junior researchers and engineers who wish to gain a better understanding of the model.
+
+**The North-Star objective of Amphion is to offer a platform for studying the conversion of any inputs into audio.** Amphion is designed to support individual generation tasks, including but not limited to,
+
+- **TTS**: Text to Speech (⛳ supported)
+- **SVS**: Singing Voice Synthesis (👨‍💻 developing)
+- **VC**: Voice Conversion (👨‍💻 developing)
+- **SVC**: Singing Voice Conversion (⛳ supported)
+- **TTA**: Text to Audio (⛳ supported)
+- **TTM**: Text to Music (👨‍💻 developing)
+- more…
+
+In addition to the specific generation tasks, Amphion includes several **vocoders** and **evaluation metrics**. A vocoder is an important module for producing high-quality audio signals, while evaluation metrics are critical for ensuring consistent metrics in generation tasks. Moreover, Amphion is dedicated to advancing audio generation in real-world applications, such as building **large-scale datasets** for speech synthesis.
+
+## 🚀 News
+- **2024/07/01**: Amphion now releases **Emilia**, the first open-source multilingual in-the-wild dataset for speech generation with over 101k hours of speech data, and the **Emilia-Pipe**, the first open-source preprocessing pipeline designed to transform in-the-wild speech data into high-quality training data with annotations for speech generation! [![arXiv](https://img.shields.io/badge/arXiv-Paper-COLOR.svg)](https://arxiv.org/abs/2407.05361) [![hf](https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace-Dataset-yellow)](https://huggingface.co/datasets/amphion/Emilia) [![demo](https://img.shields.io/badge/WebPage-Demo-red)](https://emilia-dataset.github.io/Emilia-Demo-Page/) [![readme](https://img.shields.io/badge/README-Key%20Features-blue)](preprocessors/Emilia/README.md)
+- **2024/06/17**: Amphion has a new release for its **VALL-E** model! It uses Llama as its underlying architecture and has better model performance, faster training speed, and more readable codes compared to our first version. [![readme](https://img.shields.io/badge/README-Key%20Features-blue)](egs/tts/VALLE_V2/README.md)
+- **2024/03/12**: Amphion now support **NaturalSpeech3 FACodec** and release pretrained checkpoints. [![arXiv](https://img.shields.io/badge/arXiv-Paper-COLOR.svg)](https://arxiv.org/abs/2403.03100) [![hf](https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace-model-yellow)](https://huggingface.co/amphion/naturalspeech3_facodec) [![hf](https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace-demo-pink)](https://huggingface.co/spaces/amphion/naturalspeech3_facodec) [![readme](https://img.shields.io/badge/README-Key%20Features-blue)](models/codec/ns3_codec/README.md)
+- **2024/02/22**: The first Amphion visualization tool, **SingVisio**, release. [![arXiv](https://img.shields.io/badge/arXiv-Paper-COLOR.svg)](https://arxiv.org/abs/2402.12660) [![openxlab](https://cdn-static.openxlab.org.cn/app-center/openxlab_app.svg)](https://openxlab.org.cn/apps/detail/Amphion/SingVisio) [![Video](https://img.shields.io/badge/Video-Demo-orange)](https://github.com/open-mmlab/Amphion/assets/33707885/0a6e39e8-d5f1-4288-b0f8-32da5a2d6e96) [![readme](https://img.shields.io/badge/README-Key%20Features-blue)](egs/visualization/SingVisio/README.md)
+- **2023/12/18**: Amphion v0.1 release. [![arXiv](https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg)](https://arxiv.org/abs/2312.09911) [![hf](https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace-Amphion-pink)](https://huggingface.co/amphion) [![youtube](https://img.shields.io/badge/YouTube-Demo-red)](https://www.youtube.com/watch?v=1aw0HhcggvQ) [![readme](https://img.shields.io/badge/README-Key%20Features-blue)](https://github.com/open-mmlab/Amphion/pull/39)
+- **2023/11/28**: Amphion alpha release. [![readme](https://img.shields.io/badge/README-Key%20Features-blue)](https://github.com/open-mmlab/Amphion/pull/2)
+
+## ⭐ Key Features
+
+### TTS: Text to Speech
+
+- Amphion achieves state-of-the-art performance compared to existing open-source repositories on text-to-speech (TTS) systems. It supports the following models or architectures:
+    - [FastSpeech2](https://arxiv.org/abs/2006.04558): A non-autoregressive TTS architecture that utilizes feed-forward Transformer blocks.
+    - [VITS](https://arxiv.org/abs/2106.06103): An end-to-end TTS architecture that utilizes conditional variational autoencoder with adversarial learning
+    - [VALL-E](https://arxiv.org/abs/2301.02111): A zero-shot TTS architecture that uses a neural codec language model with discrete codes.
+    - [NaturalSpeech2](https://arxiv.org/abs/2304.09116): An architecture for TTS that utilizes a latent diffusion model to generate natural-sounding voices.
+    - [Jets](Jets): An end-to-end TTS model that jointly trains FastSpeech2 and HiFi-GAN with an alignment module.
+
+### SVC: Singing Voice Conversion
+
+- Ampion supports multiple content-based features from various pretrained models, including [WeNet](https://github.com/wenet-e2e/wenet), [Whisper](https://github.com/openai/whisper), and [ContentVec](https://github.com/auspicious3000/contentvec). Their specific roles in SVC has been investigated in our NeurIPS 2023 workshop paper. [![arXiv](https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg)](https://arxiv.org/abs/2310.11160) [![code](https://img.shields.io/badge/README-Code-red)](egs/svc/MultipleContentsSVC)
+- Amphion implements several state-of-the-art model architectures, including diffusion-, transformer-, VAE- and flow-based models. The diffusion-based architecture uses [Bidirectional dilated CNN](https://openreview.net/pdf?id=a-xFK8Ymz5J) as a backend and supports several sampling algorithms such as [DDPM](https://arxiv.org/pdf/2006.11239.pdf), [DDIM](https://arxiv.org/pdf/2010.02502.pdf), and [PNDM](https://arxiv.org/pdf/2202.09778.pdf). Additionally, it supports single-step inference based on the [Consistency Model](https://openreview.net/pdf?id=FmqFfMTNnv).
+
+### TTA: Text to Audio
+
+- Amphion supports the TTA with a latent diffusion model. It is designed like [AudioLDM](https://arxiv.org/abs/2301.12503), [Make-an-Audio](https://arxiv.org/abs/2301.12661), and [AUDIT](https://arxiv.org/abs/2304.00830). It is also the official implementation of the text-to-audio generation part of our NeurIPS 2023 paper. [![arXiv](https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg)](https://arxiv.org/abs/2304.00830) [![code](https://img.shields.io/badge/README-Code-red)](egs/tta/RECIPE.md)
+
+### Vocoder
+
+- Amphion supports various widely-used neural vocoders, including:
+    - GAN-based vocoders: [MelGAN](https://arxiv.org/abs/1910.06711), [HiFi-GAN](https://arxiv.org/abs/2010.05646), [NSF-HiFiGAN](https://github.com/nii-yamagishilab/project-NN-Pytorch-scripts), [BigVGAN](https://arxiv.org/abs/2206.04658), [APNet](https://arxiv.org/abs/2305.07952).
+    - Flow-based vocoders: [WaveGlow](https://arxiv.org/abs/1811.00002).
+    - Diffusion-based vocoders: [Diffwave](https://arxiv.org/abs/2009.09761).
+    - Auto-regressive based vocoders: [WaveNet](https://arxiv.org/abs/1609.03499), [WaveRNN](https://arxiv.org/abs/1802.08435v1).
+- Amphion provides the official implementation of [Multi-Scale Constant-Q Transform Discriminator](https://arxiv.org/abs/2311.14957) (our ICASSP 2024 paper). It can be used to enhance any architecture GAN-based vocoders during training, and keep the inference stage (such as memory or speed) unchanged. [![arXiv](https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg)](https://arxiv.org/abs/2311.14957) [![code](https://img.shields.io/badge/README-Code-red)](egs/vocoder/gan/tfr_enhanced_hifigan)
+
+### Evaluation
+
+Amphion provides a comprehensive objective evaluation of the generated audio. The evaluation metrics contain:
+
+- **F0 Modeling**: F0 Pearson Coefficients, F0 Periodicity Root Mean Square Error, F0 Root Mean Square Error, Voiced/Unvoiced F1 Score, etc.
+- **Energy Modeling**: Energy Root Mean Square Error, Energy Pearson Coefficients, etc.
+- **Intelligibility**: Character/Word Error Rate, which can be calculated based on [Whisper](https://github.com/openai/whisper) and more.
+- **Spectrogram Distortion**: Frechet Audio Distance (FAD), Mel Cepstral Distortion (MCD), Multi-Resolution STFT Distance (MSTFT), Perceptual Evaluation of Speech Quality (PESQ), Short Time Objective Intelligibility (STOI), etc.
+- **Speaker Similarity**: Cosine similarity, which can be calculated based on [RawNet3](https://github.com/Jungjee/RawNet), [Resemblyzer](https://github.com/resemble-ai/Resemblyzer), [WeSpeaker](https://github.com/wenet-e2e/wespeaker), [WavLM](https://github.com/microsoft/unilm/tree/master/wavlm) and more.
+
+### Datasets
+
+- Amphion unifies the data preprocess of the open-source datasets including [AudioCaps](https://audiocaps.github.io/), [LibriTTS](https://www.openslr.org/60/), [LJSpeech](https://keithito.com/LJ-Speech-Dataset/), [M4Singer](https://github.com/M4Singer/M4Singer), [Opencpop](https://wenet.org.cn/opencpop/), [OpenSinger](https://github.com/Multi-Singer/Multi-Singer.github.io), [SVCC](http://vc-challenge.org/), [VCTK](https://datashare.ed.ac.uk/handle/10283/3443), and more. The supported dataset list can be seen [here](egs/datasets/README.md) (updating). 
+- Amphion (exclusively) supports the [**Emilia**](preprocessors/Emilia/README.md) dataset and its preprocessing pipeline **Emilia-Pipe** for in-the-wild speech data!
+
+### Visualization
+
+Amphion provides visualization tools to interactively illustrate the internal processing mechanism of classic models. This provides an invaluable resource for educational purposes and for facilitating understandable research.
+
+Currently, Amphion supports [SingVisio](egs/visualization/SingVisio/README.md), a visualization tool of the diffusion model for singing voice conversion. [![arXiv](https://img.shields.io/badge/arXiv-Paper-COLOR.svg)](https://arxiv.org/abs/2402.12660) [![openxlab](https://cdn-static.openxlab.org.cn/app-center/openxlab_app.svg)](https://openxlab.org.cn/apps/detail/Amphion/SingVisio) [![Video](https://img.shields.io/badge/Video-Demo-orange)](https://github.com/open-mmlab/Amphion/assets/33707885/0a6e39e8-d5f1-4288-b0f8-32da5a2d6e96)
+
+
+## 📀 Installation
+
+Amphion can be installed through either Setup Installer or Docker Image.
+
+### Setup Installer
+
+```bash
+git clone https://github.com/open-mmlab/Amphion.git
+cd Amphion
+
+# Install Python Environment
+conda create --name amphion python=3.9.15
+conda activate amphion
+
+# Install Python Packages Dependencies
+sh env.sh
+```
+
+### Docker Image
+
+1. Install [Docker](https://docs.docker.com/get-docker/), [NVIDIA Driver](https://www.nvidia.com/download/index.aspx), [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html), and [CUDA](https://developer.nvidia.com/cuda-downloads).
+
+2. Run the following commands:
+```bash
+git clone https://github.com/open-mmlab/Amphion.git
+cd Amphion
+
+docker pull realamphion/amphion
+docker run --runtime=nvidia --gpus all -it -v .:/app realamphion/amphion
+```
+Mount dataset by argument `-v` is necessary when using Docker. Please refer to [Mount dataset in Docker container](egs/datasets/docker.md) and [Docker Docs](https://docs.docker.com/engine/reference/commandline/container_run/#volume) for more details.
+
+
+## 🐍 Usage in Python
+
+We detail the instructions of different tasks in the following recipes:
+
+- [Text to Speech (TTS)](egs/tts/README.md)
+- [Singing Voice Conversion (SVC)](egs/svc/README.md)
+- [Text to Audio (TTA)](egs/tta/README.md)
+- [Vocoder](egs/vocoder/README.md)
+- [Evaluation](egs/metrics/README.md)
+- [Visualization](egs/visualization/README.md)
+
+## 👨‍💻 Contributing
+We appreciate all contributions to improve Amphion. Please refer to [CONTRIBUTING.md](.github/CONTRIBUTING.md) for the contributing guideline.
+
+## 🙏 Acknowledgement
+
+
+- [ming024's FastSpeech2](https://github.com/ming024/FastSpeech2) and [jaywalnut310's VITS](https://github.com/jaywalnut310/vits) for model architecture code.
+- [lifeiteng's VALL-E](https://github.com/lifeiteng/vall-e) for training pipeline and model architecture design.
+- [SpeechTokenizer](https://github.com/ZhangXInFD/SpeechTokenizer) for semantic-distilled tokenizer design.
+- [WeNet](https://github.com/wenet-e2e/wenet), [Whisper](https://github.com/openai/whisper), [ContentVec](https://github.com/auspicious3000/contentvec), and [RawNet3](https://github.com/Jungjee/RawNet) for pretrained models and inference code.
+- [HiFi-GAN](https://github.com/jik876/hifi-gan) for GAN-based Vocoder's architecture design and training strategy.
+- [Encodec](https://github.com/facebookresearch/encodec) for well-organized GAN Discriminator's architecture and basic blocks.
+- [Latent Diffusion](https://github.com/CompVis/latent-diffusion) for model architecture design.
+- [TensorFlowTTS](https://github.com/TensorSpeech/TensorFlowTTS) for preparing the MFA tools.
+
+
+## ©️ License
+
+Amphion is under the [MIT License](LICENSE). It is free for both research and commercial use cases.
+
+## 📚 Citations
+
+```bibtex
+@article{zhang2023amphion,
+      title={Amphion: An Open-Source Audio, Music and Speech Generation Toolkit}, 
+      author={Xueyao Zhang and Liumeng Xue and Yicheng Gu and Yuancheng Wang and Haorui He and Chaoren Wang and Xi Chen and Zihao Fang and Haopeng Chen and Junan Zhang and Tze Ying Tang and Lexiao Zou and Mingxuan Wang and Jun Han and Kai Chen and Haizhou Li and Zhizheng Wu},
+      journal={arXiv},
+      year={2024},
+      volume={abs/2312.09911}
+}
+```

bins/calc_metrics.py

@@ -0,0 +1,268 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+import sys
+import numpy as np
+import json
+import argparse
+import whisper
+import torch
+
+from glob import glob
+from tqdm import tqdm
+from collections import defaultdict
+
+
+from evaluation.metrics.energy.energy_rmse import extract_energy_rmse
+from evaluation.metrics.energy.energy_pearson_coefficients import (
+    extract_energy_pearson_coeffcients,
+)
+from evaluation.metrics.f0.f0_pearson_coefficients import extract_fpc
+from evaluation.metrics.f0.f0_periodicity_rmse import extract_f0_periodicity_rmse
+from evaluation.metrics.f0.f0_rmse import extract_f0rmse
+from evaluation.metrics.f0.v_uv_f1 import extract_f1_v_uv
+from evaluation.metrics.intelligibility.character_error_rate import extract_cer
+from evaluation.metrics.intelligibility.word_error_rate import extract_wer
+from evaluation.metrics.similarity.speaker_similarity import extract_similarity
+from evaluation.metrics.spectrogram.frechet_distance import extract_fad
+from evaluation.metrics.spectrogram.mel_cepstral_distortion import extract_mcd
+from evaluation.metrics.spectrogram.multi_resolution_stft_distance import extract_mstft
+from evaluation.metrics.spectrogram.pesq import extract_pesq
+from evaluation.metrics.spectrogram.scale_invariant_signal_to_distortion_ratio import (
+    extract_si_sdr,
+)
+from evaluation.metrics.spectrogram.scale_invariant_signal_to_noise_ratio import (
+    extract_si_snr,
+)
+from evaluation.metrics.spectrogram.short_time_objective_intelligibility import (
+    extract_stoi,
+)
+
+METRIC_FUNC = {
+    "energy_rmse": extract_energy_rmse,
+    "energy_pc": extract_energy_pearson_coeffcients,
+    "fpc": extract_fpc,
+    "f0_periodicity_rmse": extract_f0_periodicity_rmse,
+    "f0rmse": extract_f0rmse,
+    "v_uv_f1": extract_f1_v_uv,
+    "cer": extract_cer,
+    "wer": extract_wer,
+    "similarity": extract_similarity,
+    "fad": extract_fad,
+    "mcd": extract_mcd,
+    "mstft": extract_mstft,
+    "pesq": extract_pesq,
+    "si_sdr": extract_si_sdr,
+    "si_snr": extract_si_snr,
+    "stoi": extract_stoi,
+}
+
+
+def calc_metric(
+    ref_dir,
+    deg_dir,
+    dump_dir,
+    metrics,
+    **kwargs,
+):
+    result = defaultdict()
+
+    for metric in tqdm(metrics):
+        if metric in ["fad", "similarity"]:
+            result[metric] = str(METRIC_FUNC[metric](ref_dir, deg_dir, kwargs=kwargs))
+            continue
+
+        audios_ref = []
+        audios_deg = []
+
+        files = glob(deg_dir + "/*.wav")
+
+        for file in files:
+            audios_deg.append(file)
+            uid = file.split("/")[-1].split(".wav")[0]
+            file_gt = ref_dir + "/{}.wav".format(uid)
+            audios_ref.append(file_gt)
+
+        if metric in ["wer", "cer"] and kwargs["intelligibility_mode"] == "gt_content":
+            ltr_path = kwargs["ltr_path"]
+            tmpltrs = {}
+            with open(ltr_path, "r") as f:
+                for line in f:
+                    paras = line.replace("\n", "").split("|")
+                    paras[1] = paras[1].replace(" ", "")
+                    paras[1] = paras[1].replace(".", "")
+                    paras[1] = paras[1].replace("'", "")
+                    paras[1] = paras[1].replace("-", "")
+                    paras[1] = paras[1].replace(",", "")
+                    paras[1] = paras[1].replace("!", "")
+                    paras[1] = paras[1].lower()
+                    tmpltrs[paras[0]] = paras[1]
+            ltrs = []
+            files = glob(ref_dir + "/*.wav")
+            for file in files:
+                ltrs.append(tmpltrs[os.path.basename(file)])
+
+        if metric in ["v_uv_f1"]:
+            tp_total = 0
+            fp_total = 0
+            fn_total = 0
+
+            for i in tqdm(range(len(audios_ref))):
+                audio_ref = audios_ref[i]
+                audio_deg = audios_deg[i]
+                tp, fp, fn = METRIC_FUNC[metric](audio_ref, audio_deg, kwargs=kwargs)
+                tp_total += tp
+                fp_total += fp
+                fn_total += fn
+
+            result[metric] = str(tp_total / (tp_total + (fp_total + fn_total) / 2))
+        else:
+            scores = []
+            for i in tqdm(range(len(audios_ref))):
+                audio_ref = audios_ref[i]
+                audio_deg = audios_deg[i]
+
+                if metric in ["wer", "cer"]:
+                    model = whisper.load_model("large")
+                    mode = kwargs["intelligibility_mode"]
+                    if torch.cuda.is_available():
+                        device = torch.device("cuda")
+                        model = model.to(device)
+
+                    if mode == "gt_audio":
+                        kwargs["audio_ref"] = audio_ref
+                        kwargs["audio_deg"] = audio_deg
+                        score = METRIC_FUNC[metric](
+                            model,
+                            kwargs=kwargs,
+                        )
+                    elif mode == "gt_content":
+                        kwargs["content_gt"] = ltrs[i]
+                        kwargs["audio_deg"] = audio_deg
+                        score = METRIC_FUNC[metric](
+                            model,
+                            kwargs=kwargs,
+                        )
+                else:
+                    score = METRIC_FUNC[metric](
+                        audio_ref,
+                        audio_deg,
+                        kwargs=kwargs,
+                    )
+                if not np.isnan(score):
+                    scores.append(score)
+
+            scores = np.array(scores)
+            result["{}".format(metric)] = str(np.mean(scores))
+
+    data = json.dumps(result, indent=4)
+
+    with open(os.path.join(dump_dir, "result.json"), "w", newline="\n") as f:
+        f.write(data)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--ref_dir",
+        type=str,
+        help="Path to the reference audio folder.",
+    )
+    parser.add_argument(
+        "--deg_dir",
+        type=str,
+        help="Path to the test audio folder.",
+    )
+    parser.add_argument(
+        "--dump_dir",
+        type=str,
+        help="Path to dump the results.",
+    )
+    parser.add_argument(
+        "--metrics",
+        nargs="+",
+        help="Metrics used to evaluate.",
+    )
+    parser.add_argument(
+        "--fs",
+        type=str,
+        default="None",
+        help="(Optional) Sampling rate",
+    )
+    parser.add_argument(
+        "--align_method",
+        type=str,
+        default="dtw",
+        help="(Optional) Method for aligning feature length. ['cut', 'dtw']",
+    )
+
+    parser.add_argument(
+        "--db_scale",
+        type=str,
+        default="True",
+        help="(Optional) Wether or not computing energy related metrics in db scale.",
+    )
+    parser.add_argument(
+        "--f0_subtract_mean",
+        type=str,
+        default="True",
+        help="(Optional) Wether or not computing f0 related metrics with mean value subtracted.",
+    )
+
+    parser.add_argument(
+        "--similarity_model",
+        type=str,
+        default="wavlm",
+        help="(Optional)The model for computing speaker similarity. ['rawnet', 'wavlm', 'resemblyzer']",
+    )
+    parser.add_argument(
+        "--similarity_mode",
+        type=str,
+        default="pairwith",
+        help="(Optional)The method of calculating similarity, where set to overall means computing \
+        the speaker similarity between two folder of audios content freely, and set to pairwith means \
+        computing the speaker similarity between a seires of paired gt/pred audios",
+    )
+
+    parser.add_argument(
+        "--ltr_path",
+        type=str,
+        default="None",
+        help="(Optional)Path to the transcription file,Note that the format in the transcription \
+            file is 'file name|transcription'",
+    )
+    parser.add_argument(
+        "--intelligibility_mode",
+        type=str,
+        default="gt_audio",
+        help="(Optional)The method of calculating WER and CER, where set to gt_audio means selecting \
+        the recognition content of the reference audio as the target, and set to gt_content means \
+        using transcription as the target",
+    )
+    parser.add_argument(
+        "--language",
+        type=str,
+        default="english",
+        help="(Optional)['english','chinese']",
+    )
+
+    args = parser.parse_args()
+
+    calc_metric(
+        args.ref_dir,
+        args.deg_dir,
+        args.dump_dir,
+        args.metrics,
+        fs=int(args.fs) if args.fs != "None" else None,
+        method=args.align_method,
+        db_scale=True if args.db_scale == "True" else False,
+        need_mean=True if args.f0_subtract_mean == "True" else False,
+        model_name=args.similarity_model,
+        similarity_mode=args.similarity_mode,
+        ltr_path=args.ltr_path,
+        intelligibility_mode=args.intelligibility_mode,
+        language=args.language,
+    )

bins/svc/inference.py

@@ -0,0 +1,265 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+import os
+import glob
+from tqdm import tqdm
+import json
+import torch
+import time
+
+from models.svc.diffusion.diffusion_inference import DiffusionInference
+from models.svc.comosvc.comosvc_inference import ComoSVCInference
+from models.svc.transformer.transformer_inference import TransformerInference
+from models.svc.vits.vits_inference import VitsInference
+from utils.util import load_config
+from utils.audio_slicer import split_audio, merge_segments_encodec
+from processors import acoustic_extractor, content_extractor
+
+
+def build_inference(args, cfg, infer_type="from_dataset"):
+    supported_inference = {
+        "DiffWaveNetSVC": DiffusionInference,
+        "DiffComoSVC": ComoSVCInference,
+        "TransformerSVC": TransformerInference,
+        "VitsSVC": VitsInference,
+    }
+
+    inference_class = supported_inference[cfg.model_type]
+    return inference_class(args, cfg, infer_type)
+
+
+def prepare_for_audio_file(args, cfg, num_workers=1):
+    preprocess_path = cfg.preprocess.processed_dir
+    audio_name = cfg.inference.source_audio_name
+    temp_audio_dir = os.path.join(preprocess_path, audio_name)
+
+    ### eval file
+    t = time.time()
+    eval_file = prepare_source_eval_file(cfg, temp_audio_dir, audio_name)
+    args.source = eval_file
+    with open(eval_file, "r") as f:
+        metadata = json.load(f)
+    print("Prepare for meta eval data: {:.1f}s".format(time.time() - t))
+
+    ### acoustic features
+    t = time.time()
+    acoustic_extractor.extract_utt_acoustic_features_serial(
+        metadata, temp_audio_dir, cfg
+    )
+    if cfg.preprocess.use_min_max_norm_mel == True:
+        acoustic_extractor.cal_mel_min_max(
+            dataset=audio_name, output_path=preprocess_path, cfg=cfg, metadata=metadata
+        )
+    acoustic_extractor.cal_pitch_statistics_svc(
+        dataset=audio_name, output_path=preprocess_path, cfg=cfg, metadata=metadata
+    )
+    print("Prepare for acoustic features: {:.1f}s".format(time.time() - t))
+
+    ### content features
+    t = time.time()
+    content_extractor.extract_utt_content_features_dataloader(
+        cfg, metadata, num_workers
+    )
+    print("Prepare for content features: {:.1f}s".format(time.time() - t))
+    return args, cfg, temp_audio_dir
+
+
+def merge_for_audio_segments(audio_files, args, cfg):
+    audio_name = cfg.inference.source_audio_name
+    target_singer_name = args.target_singer
+
+    merge_segments_encodec(
+        wav_files=audio_files,
+        fs=cfg.preprocess.sample_rate,
+        output_path=os.path.join(
+            args.output_dir, "{}_{}.wav".format(audio_name, target_singer_name)
+        ),
+        overlap_duration=cfg.inference.segments_overlap_duration,
+    )
+
+    for tmp_file in audio_files:
+        os.remove(tmp_file)
+
+
+def prepare_source_eval_file(cfg, temp_audio_dir, audio_name):
+    """
+    Prepare the eval file (json) for an audio
+    """
+
+    audio_chunks_results = split_audio(
+        wav_file=cfg.inference.source_audio_path,
+        target_sr=cfg.preprocess.sample_rate,
+        output_dir=os.path.join(temp_audio_dir, "wavs"),
+        max_duration_of_segment=cfg.inference.segments_max_duration,
+        overlap_duration=cfg.inference.segments_overlap_duration,
+    )
+
+    metadata = []
+    for i, res in enumerate(audio_chunks_results):
+        res["index"] = i
+        res["Dataset"] = audio_name
+        res["Singer"] = audio_name
+        res["Uid"] = "{}_{}".format(audio_name, res["Uid"])
+        metadata.append(res)
+
+    eval_file = os.path.join(temp_audio_dir, "eval.json")
+    with open(eval_file, "w") as f:
+        json.dump(metadata, f, indent=4, ensure_ascii=False, sort_keys=True)
+
+    return eval_file
+
+
+def cuda_relevant(deterministic=False):
+    torch.cuda.empty_cache()
+    # TF32 on Ampere and above
+    torch.backends.cuda.matmul.allow_tf32 = True
+    torch.backends.cudnn.enabled = True
+    torch.backends.cudnn.allow_tf32 = True
+    # Deterministic
+    torch.backends.cudnn.deterministic = deterministic
+    torch.backends.cudnn.benchmark = not deterministic
+    torch.use_deterministic_algorithms(deterministic)
+
+
+def infer(args, cfg, infer_type):
+    # Build inference
+    t = time.time()
+    trainer = build_inference(args, cfg, infer_type)
+    print("Model Init: {:.1f}s".format(time.time() - t))
+
+    # Run inference
+    t = time.time()
+    output_audio_files = trainer.inference()
+    print("Model inference: {:.1f}s".format(time.time() - t))
+    return output_audio_files
+
+
+def build_parser():
+    r"""Build argument parser for inference.py.
+    Anything else should be put in an extra config YAML file.
+    """
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config",
+        type=str,
+        required=True,
+        help="JSON/YAML file for configurations.",
+    )
+    parser.add_argument(
+        "--acoustics_dir",
+        type=str,
+        help="Acoustics model checkpoint directory. If a directory is given, "
+        "search for the latest checkpoint dir in the directory. If a specific "
+        "checkpoint dir is given, directly load the checkpoint.",
+    )
+    parser.add_argument(
+        "--vocoder_dir",
+        type=str,
+        required=True,
+        help="Vocoder checkpoint directory. Searching behavior is the same as "
+        "the acoustics one.",
+    )
+    parser.add_argument(
+        "--target_singer",
+        type=str,
+        required=True,
+        help="convert to a specific singer (e.g. --target_singers singer_id).",
+    )
+    parser.add_argument(
+        "--trans_key",
+        default=0,
+        help="0: no pitch shift; autoshift: pitch shift;  int: key shift.",
+    )
+    parser.add_argument(
+        "--source",
+        type=str,
+        default="source_audio",
+        help="Source audio file or directory. If a JSON file is given, "
+        "inference from dataset is applied. If a directory is given, "
+        "inference from all wav/flac/mp3 audio files in the directory is applied. "
+        "Default: inference from all wav/flac/mp3 audio files in ./source_audio",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="conversion_results",
+        help="Output directory. Default: ./conversion_results",
+    )
+    parser.add_argument(
+        "--log_level",
+        type=str,
+        default="warning",
+        help="Logging level. Default: warning",
+    )
+    parser.add_argument(
+        "--keep_cache",
+        action="store_true",
+        default=True,
+        help="Keep cache files. Only applicable to inference from files.",
+    )
+    parser.add_argument(
+        "--diffusion_inference_steps",
+        type=int,
+        default=1000,
+        help="Number of inference steps. Only applicable to diffusion inference.",
+    )
+    return parser
+
+
+def main():
+    ### Parse arguments and config
+    args = build_parser().parse_args()
+    cfg = load_config(args.config)
+
+    # CUDA settings
+    cuda_relevant()
+
+    if os.path.isdir(args.source):
+        ### Infer from file
+
+        # Get all the source audio files (.wav, .flac, .mp3)
+        source_audio_dir = args.source
+        audio_list = []
+        for suffix in ["wav", "flac", "mp3"]:
+            audio_list += glob.glob(
+                os.path.join(source_audio_dir, "**/*.{}".format(suffix)), recursive=True
+            )
+        print("There are {} source audios: ".format(len(audio_list)))
+
+        # Infer for every file as dataset
+        output_root_path = args.output_dir
+        for audio_path in tqdm(audio_list):
+            audio_name = audio_path.split("/")[-1].split(".")[0]
+            args.output_dir = os.path.join(output_root_path, audio_name)
+            print("\n{}\nConversion for {}...\n".format("*" * 10, audio_name))
+
+            cfg.inference.source_audio_path = audio_path
+            cfg.inference.source_audio_name = audio_name
+            cfg.inference.segments_max_duration = 10.0
+            cfg.inference.segments_overlap_duration = 1.0
+
+            # Prepare metadata and features
+            args, cfg, cache_dir = prepare_for_audio_file(args, cfg)
+
+            # Infer from file
+            output_audio_files = infer(args, cfg, infer_type="from_file")
+
+            # Merge the split segments
+            merge_for_audio_segments(output_audio_files, args, cfg)
+
+            # Keep or remove caches
+            if not args.keep_cache:
+                os.removedirs(cache_dir)
+
+    else:
+        ### Infer from dataset
+        infer(args, cfg, infer_type="from_dataset")
+
+
+if __name__ == "__main__":
+    main()

bins/svc/preprocess.py

@@ -0,0 +1,183 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import faulthandler
+
+faulthandler.enable()
+
+import os
+import argparse
+import json
+from multiprocessing import cpu_count
+
+
+from utils.util import load_config
+from preprocessors.processor import preprocess_dataset
+from preprocessors.metadata import cal_metadata
+from processors import acoustic_extractor, content_extractor, data_augment
+
+
+def extract_acoustic_features(dataset, output_path, cfg, n_workers=1):
+    """Extract acoustic features of utterances in the dataset
+
+    Args:
+        dataset (str): name of dataset, e.g. opencpop
+        output_path (str): directory that stores train, test and feature files of datasets
+        cfg (dict): dictionary that stores configurations
+        n_workers (int, optional): num of processes to extract features in parallel. Defaults to 1.
+    """
+    types = ["train", "test"] if "eval" not in dataset else ["test"]
+    metadata = []
+    dataset_output = os.path.join(output_path, dataset)
+
+    for dataset_type in types:
+        dataset_file = os.path.join(dataset_output, "{}.json".format(dataset_type))
+        with open(dataset_file, "r") as f:
+            metadata.extend(json.load(f))
+
+        # acoustic_extractor.extract_utt_acoustic_features_parallel(
+        #     metadata, dataset_output, cfg, n_workers=n_workers
+        # )
+    acoustic_extractor.extract_utt_acoustic_features_serial(
+        metadata, dataset_output, cfg
+    )
+
+
+def extract_content_features(dataset, output_path, cfg, num_workers=1):
+    """Extract content features of utterances in the dataset
+
+    Args:
+        dataset (str): name of dataset, e.g. opencpop
+        output_path (str): directory that stores train, test and feature files of datasets
+        cfg (dict): dictionary that stores configurations
+    """
+    types = ["train", "test"] if "eval" not in dataset else ["test"]
+    metadata = []
+    for dataset_type in types:
+        dataset_output = os.path.join(output_path, dataset)
+        dataset_file = os.path.join(dataset_output, "{}.json".format(dataset_type))
+        with open(dataset_file, "r") as f:
+            metadata.extend(json.load(f))
+
+    content_extractor.extract_utt_content_features_dataloader(
+        cfg, metadata, num_workers
+    )
+
+
+def preprocess(cfg, args):
+    """Proprocess raw data of single or multiple datasets (in cfg.dataset)
+
+    Args:
+        cfg (dict): dictionary that stores configurations
+        args (ArgumentParser): specify the configuration file and num_workers
+    """
+    # Specify the output root path to save the processed data
+    output_path = cfg.preprocess.processed_dir
+    os.makedirs(output_path, exist_ok=True)
+
+    ## Split train and test sets
+    for dataset in cfg.dataset:
+        print("Preprocess {}...".format(dataset))
+        preprocess_dataset(
+            dataset,
+            cfg.dataset_path[dataset],
+            output_path,
+            cfg.preprocess,
+            cfg.task_type,
+            is_custom_dataset=dataset in cfg.use_custom_dataset,
+        )
+
+    # Data augmentation: create new wav files with pitch shift, formant shift, equalizer, time stretch
+    try:
+        assert isinstance(
+            cfg.preprocess.data_augment, list
+        ), "Please provide a list of datasets need to be augmented."
+        if len(cfg.preprocess.data_augment) > 0:
+            new_datasets_list = []
+            for dataset in cfg.preprocess.data_augment:
+                new_datasets = data_augment.augment_dataset(cfg, dataset)
+                new_datasets_list.extend(new_datasets)
+            cfg.dataset.extend(new_datasets_list)
+            print("Augmentation datasets: ", cfg.dataset)
+    except:
+        print("No Data Augmentation.")
+
+    # Dump metadata of datasets (singers, train/test durations, etc.)
+    cal_metadata(cfg)
+
+    ## Prepare the acoustic features
+    for dataset in cfg.dataset:
+        # Skip augmented datasets which do not need to extract acoustic features
+        # We will copy acoustic features from the original dataset later
+        if (
+            "pitch_shift" in dataset
+            or "formant_shift" in dataset
+            or "equalizer" in dataset in dataset
+        ):
+            continue
+        print(
+            "Extracting acoustic features for {} using {} workers ...".format(
+                dataset, args.num_workers
+            )
+        )
+        extract_acoustic_features(dataset, output_path, cfg, args.num_workers)
+        # Calculate the statistics of acoustic features
+        if cfg.preprocess.mel_min_max_norm:
+            acoustic_extractor.cal_mel_min_max(dataset, output_path, cfg)
+
+        if cfg.preprocess.extract_pitch:
+            acoustic_extractor.cal_pitch_statistics_svc(dataset, output_path, cfg)
+
+    # Copy acoustic features for augmented datasets by creating soft-links
+    for dataset in cfg.dataset:
+        if "pitch_shift" in dataset:
+            src_dataset = dataset.replace("_pitch_shift", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        elif "formant_shift" in dataset:
+            src_dataset = dataset.replace("_formant_shift", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        elif "equalizer" in dataset:
+            src_dataset = dataset.replace("_equalizer", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        else:
+            continue
+        dataset_dir = os.path.join(output_path, dataset)
+        metadata = []
+        for split in ["train", "test"] if not "eval" in dataset else ["test"]:
+            metadata_file_path = os.path.join(src_dataset_dir, "{}.json".format(split))
+            with open(metadata_file_path, "r") as f:
+                metadata.extend(json.load(f))
+        print("Copying acoustic features for {}...".format(dataset))
+        acoustic_extractor.copy_acoustic_features(
+            metadata, dataset_dir, src_dataset_dir, cfg
+        )
+        if cfg.preprocess.mel_min_max_norm:
+            acoustic_extractor.cal_mel_min_max(dataset, output_path, cfg)
+
+        if cfg.preprocess.extract_pitch:
+            acoustic_extractor.cal_pitch_statistics(dataset, output_path, cfg)
+
+    # Prepare the content features
+    for dataset in cfg.dataset:
+        print("Extracting content features for {}...".format(dataset))
+        extract_content_features(dataset, output_path, cfg, args.num_workers)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config", default="config.json", help="json files for configurations."
+    )
+    parser.add_argument("--num_workers", type=int, default=int(cpu_count()))
+    parser.add_argument("--prepare_alignment", type=bool, default=False)
+
+    args = parser.parse_args()
+    cfg = load_config(args.config)
+
+    preprocess(cfg, args)
+
+
+if __name__ == "__main__":
+    main()

bins/svc/train.py

@@ -0,0 +1,111 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+
+import torch
+
+from models.svc.diffusion.diffusion_trainer import DiffusionTrainer
+from models.svc.comosvc.comosvc_trainer import ComoSVCTrainer
+from models.svc.transformer.transformer_trainer import TransformerTrainer
+from models.svc.vits.vits_trainer import VitsSVCTrainer
+from utils.util import load_config
+
+
+def build_trainer(args, cfg):
+    supported_trainer = {
+        "DiffWaveNetSVC": DiffusionTrainer,
+        "DiffComoSVC": ComoSVCTrainer,
+        "TransformerSVC": TransformerTrainer,
+        "VitsSVC": VitsSVCTrainer,
+    }
+
+    trainer_class = supported_trainer[cfg.model_type]
+    trainer = trainer_class(args, cfg)
+    return trainer
+
+
+def cuda_relevant(deterministic=False):
+    torch.cuda.empty_cache()
+    # TF32 on Ampere and above
+    torch.backends.cuda.matmul.allow_tf32 = True
+    torch.backends.cudnn.enabled = True
+    torch.backends.cudnn.allow_tf32 = True
+    # Deterministic
+    torch.backends.cudnn.deterministic = deterministic
+    torch.backends.cudnn.benchmark = not deterministic
+    torch.use_deterministic_algorithms(deterministic)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config",
+        default="config.json",
+        help="json files for configurations.",
+        required=True,
+    )
+    parser.add_argument(
+        "--exp_name",
+        type=str,
+        default="exp_name",
+        help="A specific name to note the experiment",
+        required=True,
+    )
+    parser.add_argument(
+        "--resume",
+        action="store_true",
+        help="If specified, to resume from the existing checkpoint.",
+    )
+    parser.add_argument(
+        "--resume_from_ckpt_path",
+        type=str,
+        default="",
+        help="The specific checkpoint path that you want to resume from.",
+    )
+    parser.add_argument(
+        "--resume_type",
+        type=str,
+        default="",
+        help="`resume` for loading all the things (including model weights, optimizer, scheduler, and random states). `finetune` for loading only the model weights",
+    )
+
+    parser.add_argument(
+        "--log_level", default="warning", help="logging level (debug, info, warning)"
+    )
+    args = parser.parse_args()
+    cfg = load_config(args.config)
+
+    # Data Augmentation
+    if (
+        type(cfg.preprocess.data_augment) == list
+        and len(cfg.preprocess.data_augment) > 0
+    ):
+        new_datasets_list = []
+        for dataset in cfg.preprocess.data_augment:
+            new_datasets = [
+                f"{dataset}_pitch_shift" if cfg.preprocess.use_pitch_shift else None,
+                (
+                    f"{dataset}_formant_shift"
+                    if cfg.preprocess.use_formant_shift
+                    else None
+                ),
+                f"{dataset}_equalizer" if cfg.preprocess.use_equalizer else None,
+                f"{dataset}_time_stretch" if cfg.preprocess.use_time_stretch else None,
+            ]
+            new_datasets_list.extend(filter(None, new_datasets))
+        cfg.dataset.extend(new_datasets_list)
+
+    # CUDA settings
+    cuda_relevant()
+
+    # Build trainer
+    trainer = build_trainer(args, cfg)
+
+    trainer.train_loop()
+
+
+if __name__ == "__main__":
+    main()

bins/tta/inference.py

@@ -0,0 +1,94 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+from argparse import ArgumentParser
+import os
+
+from models.tta.ldm.audioldm_inference import AudioLDMInference
+from utils.util import save_config, load_model_config, load_config
+import numpy as np
+import torch
+
+
+def build_inference(args, cfg):
+    supported_inference = {
+        "AudioLDM": AudioLDMInference,
+    }
+
+    inference_class = supported_inference[cfg.model_type]
+    inference = inference_class(args, cfg)
+    return inference
+
+
+def build_parser():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--config",
+        type=str,
+        required=True,
+        help="JSON/YAML file for configurations.",
+    )
+    parser.add_argument(
+        "--text",
+        help="Text to be synthesized",
+        type=str,
+        default="Text to be synthesized.",
+    )
+    parser.add_argument(
+        "--checkpoint_path",
+        type=str,
+    )
+    parser.add_argument(
+        "--vocoder_path", type=str, help="Checkpoint path of the vocoder"
+    )
+    parser.add_argument(
+        "--vocoder_config_path", type=str, help="Config path of the vocoder"
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default=None,
+        help="Output dir for saving generated results",
+    )
+    parser.add_argument(
+        "--num_steps",
+        type=int,
+        default=200,
+        help="The total number of denosing steps",
+    )
+    parser.add_argument(
+        "--guidance_scale",
+        type=float,
+        default=4.0,
+        help="The scale of classifer free guidance",
+    )
+    parser.add_argument("--local_rank", default=-1, type=int)
+    return parser
+
+
+def main():
+    # Parse arguments
+    args = build_parser().parse_args()
+    # args, infer_type = formulate_parser(args)
+
+    # Parse config
+    cfg = load_config(args.config)
+    if torch.cuda.is_available():
+        args.local_rank = torch.device("cuda")
+    else:
+        args.local_rank = torch.device("cpu")
+    print("args: ", args)
+
+    # Build inference
+    inferencer = build_inference(args, cfg)
+
+    # Run inference
+    inferencer.inference()
+
+
+if __name__ == "__main__":
+    main()

bins/tta/preprocess.py

@@ -0,0 +1,195 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import faulthandler
+
+faulthandler.enable()
+
+import os
+import argparse
+import json
+import pyworld as pw
+from multiprocessing import cpu_count
+
+
+from utils.util import load_config
+from preprocessors.processor import preprocess_dataset, prepare_align
+from preprocessors.metadata import cal_metadata
+from processors import acoustic_extractor, content_extractor, data_augment
+
+
+def extract_acoustic_features(dataset, output_path, cfg, n_workers=1):
+    """Extract acoustic features of utterances in the dataset
+
+    Args:
+        dataset (str): name of dataset, e.g. opencpop
+        output_path (str): directory that stores train, test and feature files of datasets
+        cfg (dict): dictionary that stores configurations
+        n_workers (int, optional): num of processes to extract features in parallel. Defaults to 1.
+    """
+    types = ["train", "test"] if "eval" not in dataset else ["test"]
+    metadata = []
+    for dataset_type in types:
+        dataset_output = os.path.join(output_path, dataset)
+        dataset_file = os.path.join(dataset_output, "{}.json".format(dataset_type))
+        with open(dataset_file, "r") as f:
+            metadata.extend(json.load(f))
+
+        # acoustic_extractor.extract_utt_acoustic_features_parallel(
+        #     metadata, dataset_output, cfg, n_workers=n_workers
+        # )
+    acoustic_extractor.extract_utt_acoustic_features_serial(
+        metadata, dataset_output, cfg
+    )
+
+
+def extract_content_features(dataset, output_path, cfg, num_workers=1):
+    """Extract content features of utterances in the dataset
+
+    Args:
+        dataset (str): name of dataset, e.g. opencpop
+        output_path (str): directory that stores train, test and feature files of datasets
+        cfg (dict): dictionary that stores configurations
+    """
+    types = ["train", "test"] if "eval" not in dataset else ["test"]
+    metadata = []
+    for dataset_type in types:
+        dataset_output = os.path.join(output_path, dataset)
+        dataset_file = os.path.join(dataset_output, "{}.json".format(dataset_type))
+        with open(dataset_file, "r") as f:
+            metadata.extend(json.load(f))
+
+    content_extractor.extract_utt_content_features_dataloader(
+        cfg, metadata, num_workers
+    )
+
+
+def preprocess(cfg, args):
+    """Proprocess raw data of single or multiple datasets (in cfg.dataset)
+
+    Args:
+        cfg (dict): dictionary that stores configurations
+        args (ArgumentParser): specify the configuration file and num_workers
+    """
+    # Specify the output root path to save the processed data
+    output_path = cfg.preprocess.processed_dir
+    os.makedirs(output_path, exist_ok=True)
+
+    ## Split train and test sets
+    for dataset in cfg.dataset:
+        print("Preprocess {}...".format(dataset))
+
+        if args.prepare_alignment:
+            ## Prepare alignment with MFA
+            print("Prepare alignment {}...".format(dataset))
+            prepare_align(
+                dataset, cfg.dataset_path[dataset], cfg.preprocess, output_path
+            )
+        preprocess_dataset(
+            dataset,
+            cfg.dataset_path[dataset],
+            output_path,
+            cfg.preprocess,
+            cfg.task_type,
+            is_custom_dataset=dataset in cfg.use_custom_dataset,
+        )
+
+    # Data augmentation: create new wav files with pitch shift, formant shift, equalizer, time stretch
+    try:
+        assert isinstance(
+            cfg.preprocess.data_augment, list
+        ), "Please provide a list of datasets need to be augmented."
+        if len(cfg.preprocess.data_augment) > 0:
+            new_datasets_list = []
+            for dataset in cfg.preprocess.data_augment:
+                new_datasets = data_augment.augment_dataset(cfg, dataset)
+                new_datasets_list.extend(new_datasets)
+            cfg.dataset.extend(new_datasets_list)
+            print("Augmentation datasets: ", cfg.dataset)
+    except:
+        print("No Data Augmentation.")
+
+    # Dump metadata of datasets (singers, train/test durations, etc.)
+    cal_metadata(cfg)
+
+    ## Prepare the acoustic features
+    for dataset in cfg.dataset:
+        # Skip augmented datasets which do not need to extract acoustic features
+        # We will copy acoustic features from the original dataset later
+        if (
+            "pitch_shift" in dataset
+            or "formant_shift" in dataset
+            or "equalizer" in dataset in dataset
+        ):
+            continue
+        print(
+            "Extracting acoustic features for {} using {} workers ...".format(
+                dataset, args.num_workers
+            )
+        )
+        extract_acoustic_features(dataset, output_path, cfg, args.num_workers)
+        # Calculate the statistics of acoustic features
+        if cfg.preprocess.mel_min_max_norm:
+            acoustic_extractor.cal_mel_min_max(dataset, output_path, cfg)
+
+        if cfg.preprocess.extract_pitch:
+            acoustic_extractor.cal_pitch_statistics(dataset, output_path, cfg)
+        if cfg.preprocess.extract_energy:
+            acoustic_extractor.cal_energy_statistics(dataset, output_path, cfg)
+
+    if cfg.preprocess.align_mel_duration:
+        acoustic_extractor.align_duration_mel(dataset, output_path, cfg)
+
+    # Copy acoustic features for augmented datasets by creating soft-links
+    for dataset in cfg.dataset:
+        if "pitch_shift" in dataset:
+            src_dataset = dataset.replace("_pitch_shift", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        elif "formant_shift" in dataset:
+            src_dataset = dataset.replace("_formant_shift", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        elif "equalizer" in dataset:
+            src_dataset = dataset.replace("_equalizer", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        else:
+            continue
+        dataset_dir = os.path.join(output_path, dataset)
+        metadata = []
+        for split in ["train", "test"] if not "eval" in dataset else ["test"]:
+            metadata_file_path = os.path.join(src_dataset_dir, "{}.json".format(split))
+            with open(metadata_file_path, "r") as f:
+                metadata.extend(json.load(f))
+        print("Copying acoustic features for {}...".format(dataset))
+        acoustic_extractor.copy_acoustic_features(
+            metadata, dataset_dir, src_dataset_dir, cfg
+        )
+        if cfg.preprocess.mel_min_max_norm:
+            acoustic_extractor.cal_mel_min_max(dataset, output_path, cfg)
+
+        if cfg.preprocess.extract_pitch:
+            acoustic_extractor.cal_pitch_statistics(dataset, output_path, cfg)
+
+    # Prepare the content features
+    for dataset in cfg.dataset:
+        print("Extracting content features for {}...".format(dataset))
+        extract_content_features(dataset, output_path, cfg, args.num_workers)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config", default="config.json", help="json files for configurations."
+    )
+    parser.add_argument("--num_workers", type=int, default=int(cpu_count()))
+    parser.add_argument("--prepare_alignment", type=bool, default=False)
+
+    args = parser.parse_args()
+    cfg = load_config(args.config)
+
+    preprocess(cfg, args)
+
+
+if __name__ == "__main__":
+    main()

bins/tta/train_tta.py

@@ -0,0 +1,77 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+import os
+import torch
+
+from models.tta.autoencoder.autoencoder_trainer import AutoencoderKLTrainer
+from models.tta.ldm.audioldm_trainer import AudioLDMTrainer
+from utils.util import load_config
+
+
+def build_trainer(args, cfg):
+    supported_trainer = {
+        "AutoencoderKL": AutoencoderKLTrainer,
+        "AudioLDM": AudioLDMTrainer,
+    }
+
+    trainer_class = supported_trainer[cfg.model_type]
+    trainer = trainer_class(args, cfg)
+    return trainer
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config",
+        default="config.json",
+        help="json files for configurations.",
+        required=True,
+    )
+    parser.add_argument(
+        "--num_workers", type=int, default=6, help="Number of dataloader workers."
+    )
+    parser.add_argument(
+        "--exp_name",
+        type=str,
+        default="exp_name",
+        help="A specific name to note the experiment",
+        required=True,
+    )
+    parser.add_argument(
+        "--resume",
+        type=str,
+        default=None,
+        # action="store_true",
+        help="The model name to restore",
+    )
+    parser.add_argument(
+        "--log_level", default="info", help="logging level (info, debug, warning)"
+    )
+    parser.add_argument("--stdout_interval", default=5, type=int)
+    parser.add_argument("--local_rank", default=-1, type=int)
+    args = parser.parse_args()
+    cfg = load_config(args.config)
+    cfg.exp_name = args.exp_name
+
+    # Model saving dir
+    args.log_dir = os.path.join(cfg.log_dir, args.exp_name)
+    os.makedirs(args.log_dir, exist_ok=True)
+
+    if not cfg.train.ddp:
+        args.local_rank = torch.device("cuda")
+
+    # Build trainer
+    trainer = build_trainer(args, cfg)
+
+    # Restore models
+    if args.resume:
+        trainer.restore()
+    trainer.train()
+
+
+if __name__ == "__main__":
+    main()

bins/tts/inference.py

@@ -0,0 +1,169 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+from argparse import ArgumentParser
+import os
+
+from models.tts.fastspeech2.fs2_inference import FastSpeech2Inference
+from models.tts.vits.vits_inference import VitsInference
+from models.tts.valle.valle_inference import VALLEInference
+from models.tts.naturalspeech2.ns2_inference import NS2Inference
+from models.tts.jets.jets_inference import JetsInference
+from utils.util import load_config
+import torch
+
+
+def build_inference(args, cfg):
+    supported_inference = {
+        "FastSpeech2": FastSpeech2Inference,
+        "VITS": VitsInference,
+        "VALLE": VALLEInference,
+        "NaturalSpeech2": NS2Inference,
+        "Jets": JetsInference,
+    }
+
+    inference_class = supported_inference[cfg.model_type]
+    inference = inference_class(args, cfg)
+    return inference
+
+
+def cuda_relevant(deterministic=False):
+    torch.cuda.empty_cache()
+    # TF32 on Ampere and above
+    torch.backends.cuda.matmul.allow_tf32 = True
+    torch.backends.cudnn.enabled = True
+    torch.backends.cudnn.allow_tf32 = True
+    # Deterministic
+    torch.backends.cudnn.deterministic = deterministic
+    torch.backends.cudnn.benchmark = not deterministic
+    torch.use_deterministic_algorithms(deterministic)
+
+
+def build_parser():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--config",
+        type=str,
+        required=True,
+        help="JSON/YAML file for configurations.",
+    )
+    parser.add_argument(
+        "--dataset",
+        type=str,
+        help="convert from the source data",
+        default=None,
+    )
+    parser.add_argument(
+        "--testing_set",
+        type=str,
+        help="train, test, golden_test",
+        default="test",
+    )
+    parser.add_argument(
+        "--test_list_file",
+        type=str,
+        help="convert from the test list file",
+        default=None,
+    )
+    parser.add_argument(
+        "--speaker_name",
+        type=str,
+        default=None,
+        help="speaker name for multi-speaker synthesis, for single-sentence mode only",
+    )
+    parser.add_argument(
+        "--text",
+        help="Text to be synthesized.",
+        type=str,
+        default="",
+    )
+    parser.add_argument(
+        "--vocoder_dir",
+        type=str,
+        default=None,
+        help="Vocoder checkpoint directory. Searching behavior is the same as "
+        "the acoustics one.",
+    )
+    parser.add_argument(
+        "--acoustics_dir",
+        type=str,
+        default=None,
+        help="Acoustic model checkpoint directory. If a directory is given, "
+        "search for the latest checkpoint dir in the directory. If a specific "
+        "checkpoint dir is given, directly load the checkpoint.",
+    )
+    parser.add_argument(
+        "--checkpoint_path",
+        type=str,
+        default=None,
+        help="Acoustic model checkpoint directory. If a directory is given, "
+        "search for the latest checkpoint dir in the directory. If a specific "
+        "checkpoint dir is given, directly load the checkpoint.",
+    )
+    parser.add_argument(
+        "--mode",
+        type=str,
+        choices=["batch", "single"],
+        required=True,
+        help="Synthesize a whole dataset or a single sentence",
+    )
+    parser.add_argument(
+        "--log_level",
+        type=str,
+        default="warning",
+        help="Logging level. Default: warning",
+    )
+    parser.add_argument(
+        "--pitch_control",
+        type=float,
+        default=1.0,
+        help="control the pitch of the whole utterance, larger value for higher pitch",
+    )
+    parser.add_argument(
+        "--energy_control",
+        type=float,
+        default=1.0,
+        help="control the energy of the whole utterance, larger value for larger volume",
+    )
+    parser.add_argument(
+        "--duration_control",
+        type=float,
+        default=1.0,
+        help="control the speed of the whole utterance, larger value for slower speaking rate",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default=None,
+        help="Output dir for saving generated results",
+    )
+    return parser
+
+
+def main():
+    # Parse arguments
+    parser = build_parser()
+    VALLEInference.add_arguments(parser)
+    NS2Inference.add_arguments(parser)
+    args = parser.parse_args()
+    print(args)
+
+    # Parse config
+    cfg = load_config(args.config)
+
+    # CUDA settings
+    cuda_relevant()
+
+    # Build inference
+    inferencer = build_inference(args, cfg)
+
+    # Run inference
+    inferencer.inference()
+
+
+if __name__ == "__main__":
+    main()

bins/tts/preprocess.py

@@ -0,0 +1,244 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import faulthandler
+
+faulthandler.enable()
+
+import os
+import argparse
+import json
+import pyworld as pw
+from multiprocessing import cpu_count
+
+
+from utils.util import load_config
+from preprocessors.processor import preprocess_dataset, prepare_align
+from preprocessors.metadata import cal_metadata
+from processors import (
+    acoustic_extractor,
+    content_extractor,
+    data_augment,
+    phone_extractor,
+)
+
+
+def extract_acoustic_features(dataset, output_path, cfg, dataset_types, n_workers=1):
+    """Extract acoustic features of utterances in the dataset
+
+    Args:
+        dataset (str): name of dataset, e.g. opencpop
+        output_path (str): directory that stores train, test and feature files of datasets
+        cfg (dict): dictionary that stores configurations
+        n_workers (int, optional): num of processes to extract features in parallel. Defaults to 1.
+    """
+
+    metadata = []
+    for dataset_type in dataset_types:
+        dataset_output = os.path.join(output_path, dataset)
+        dataset_file = os.path.join(dataset_output, "{}.json".format(dataset_type))
+        with open(dataset_file, "r") as f:
+            metadata.extend(json.load(f))
+
+        # acoustic_extractor.extract_utt_acoustic_features_parallel(
+        #     metadata, dataset_output, cfg, n_workers=n_workers
+        # )
+    acoustic_extractor.extract_utt_acoustic_features_serial(
+        metadata, dataset_output, cfg
+    )
+
+
+def extract_content_features(dataset, output_path, cfg, dataset_types, num_workers=1):
+    """Extract content features of utterances in the dataset
+
+    Args:
+        dataset (str): name of dataset, e.g. opencpop
+        output_path (str): directory that stores train, test and feature files of datasets
+        cfg (dict): dictionary that stores configurations
+    """
+
+    metadata = []
+    for dataset_type in dataset_types:
+        dataset_output = os.path.join(output_path, dataset)
+        # dataset_file = os.path.join(dataset_output, "{}.json".format(dataset_type))
+        dataset_file = os.path.join(dataset_output, "{}.json".format(dataset_type))
+        with open(dataset_file, "r") as f:
+            metadata.extend(json.load(f))
+
+    content_extractor.extract_utt_content_features_dataloader(
+        cfg, metadata, num_workers
+    )
+
+
+def extract_phonme_sequences(dataset, output_path, cfg, dataset_types):
+    """Extract phoneme features of utterances in the dataset
+
+    Args:
+        dataset (str): name of dataset, e.g. opencpop
+        output_path (str): directory that stores train, test and feature files of datasets
+        cfg (dict): dictionary that stores configurations
+
+    """
+
+    metadata = []
+    for dataset_type in dataset_types:
+        dataset_output = os.path.join(output_path, dataset)
+        dataset_file = os.path.join(dataset_output, "{}.json".format(dataset_type))
+        with open(dataset_file, "r") as f:
+            metadata.extend(json.load(f))
+    phone_extractor.extract_utt_phone_sequence(dataset, cfg, metadata)
+
+
+def preprocess(cfg, args):
+    """Preprocess raw data of single or multiple datasets (in cfg.dataset)
+
+    Args:
+        cfg (dict): dictionary that stores configurations
+        args (ArgumentParser): specify the configuration file and num_workers
+    """
+    # Specify the output root path to save the processed data
+    output_path = cfg.preprocess.processed_dir
+    os.makedirs(output_path, exist_ok=True)
+
+    # Split train and test sets
+    for dataset in cfg.dataset:
+        print("Preprocess {}...".format(dataset))
+
+        if args.prepare_alignment:
+            # Prepare alignment with MFA
+            print("Prepare alignment {}...".format(dataset))
+            prepare_align(
+                dataset, cfg.dataset_path[dataset], cfg.preprocess, output_path
+            )
+
+        preprocess_dataset(
+            dataset,
+            cfg.dataset_path[dataset],
+            output_path,
+            cfg.preprocess,
+            cfg.task_type,
+            is_custom_dataset=dataset in cfg.use_custom_dataset,
+        )
+
+    # Data augmentation: create new wav files with pitch shift, formant shift, equalizer, time stretch
+    try:
+        assert isinstance(
+            cfg.preprocess.data_augment, list
+        ), "Please provide a list of datasets need to be augmented."
+        if len(cfg.preprocess.data_augment) > 0:
+            new_datasets_list = []
+            for dataset in cfg.preprocess.data_augment:
+                new_datasets = data_augment.augment_dataset(cfg, dataset)
+                new_datasets_list.extend(new_datasets)
+            cfg.dataset.extend(new_datasets_list)
+            print("Augmentation datasets: ", cfg.dataset)
+    except:
+        print("No Data Augmentation.")
+
+    # json files
+    dataset_types = list()
+    dataset_types.append((cfg.preprocess.train_file).split(".")[0])
+    dataset_types.append((cfg.preprocess.valid_file).split(".")[0])
+    if "test" not in dataset_types:
+        dataset_types.append("test")
+    if "eval" in dataset:
+        dataset_types = ["test"]
+
+    # Dump metadata of datasets (singers, train/test durations, etc.)
+    cal_metadata(cfg, dataset_types)
+
+    # Prepare the acoustic features
+    for dataset in cfg.dataset:
+        # Skip augmented datasets which do not need to extract acoustic features
+        # We will copy acoustic features from the original dataset later
+        if (
+            "pitch_shift" in dataset
+            or "formant_shift" in dataset
+            or "equalizer" in dataset in dataset
+        ):
+            continue
+        print(
+            "Extracting acoustic features for {} using {} workers ...".format(
+                dataset, args.num_workers
+            )
+        )
+        extract_acoustic_features(
+            dataset, output_path, cfg, dataset_types, args.num_workers
+        )
+        # Calculate the statistics of acoustic features
+        if cfg.preprocess.mel_min_max_norm:
+            acoustic_extractor.cal_mel_min_max(dataset, output_path, cfg)
+
+        if cfg.preprocess.extract_pitch:
+            acoustic_extractor.cal_pitch_statistics(dataset, output_path, cfg)
+
+        if cfg.preprocess.extract_energy:
+            acoustic_extractor.cal_energy_statistics(dataset, output_path, cfg)
+
+        if cfg.preprocess.pitch_norm:
+            acoustic_extractor.normalize(dataset, cfg.preprocess.pitch_dir, cfg)
+
+        if cfg.preprocess.energy_norm:
+            acoustic_extractor.normalize(dataset, cfg.preprocess.energy_dir, cfg)
+
+    # Copy acoustic features for augmented datasets by creating soft-links
+    for dataset in cfg.dataset:
+        if "pitch_shift" in dataset:
+            src_dataset = dataset.replace("_pitch_shift", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        elif "formant_shift" in dataset:
+            src_dataset = dataset.replace("_formant_shift", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        elif "equalizer" in dataset:
+            src_dataset = dataset.replace("_equalizer", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        else:
+            continue
+        dataset_dir = os.path.join(output_path, dataset)
+        metadata = []
+        for split in ["train", "test"] if not "eval" in dataset else ["test"]:
+            metadata_file_path = os.path.join(src_dataset_dir, "{}.json".format(split))
+            with open(metadata_file_path, "r") as f:
+                metadata.extend(json.load(f))
+        print("Copying acoustic features for {}...".format(dataset))
+        acoustic_extractor.copy_acoustic_features(
+            metadata, dataset_dir, src_dataset_dir, cfg
+        )
+        if cfg.preprocess.mel_min_max_norm:
+            acoustic_extractor.cal_mel_min_max(dataset, output_path, cfg)
+
+        if cfg.preprocess.extract_pitch:
+            acoustic_extractor.cal_pitch_statistics(dataset, output_path, cfg)
+
+    # Prepare the content features
+    for dataset in cfg.dataset:
+        print("Extracting content features for {}...".format(dataset))
+        extract_content_features(
+            dataset, output_path, cfg, dataset_types, args.num_workers
+        )
+
+    # Prepare the phenome squences
+    if cfg.preprocess.extract_phone:
+        for dataset in cfg.dataset:
+            print("Extracting phoneme sequence for {}...".format(dataset))
+            extract_phonme_sequences(dataset, output_path, cfg, dataset_types)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config", default="config.json", help="json files for configurations."
+    )
+    parser.add_argument("--num_workers", type=int, default=int(cpu_count()))
+    parser.add_argument("--prepare_alignment", type=bool, default=False)
+
+    args = parser.parse_args()
+    cfg = load_config(args.config)
+
+    preprocess(cfg, args)
+
+
+if __name__ == "__main__":
+    main()

bins/tts/train.py

@@ -0,0 +1,152 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+
+import torch
+
+from models.tts.fastspeech2.fs2_trainer import FastSpeech2Trainer
+from models.tts.vits.vits_trainer import VITSTrainer
+from models.tts.valle.valle_trainer import VALLETrainer
+from models.tts.naturalspeech2.ns2_trainer import NS2Trainer
+from models.tts.valle_v2.valle_ar_trainer import ValleARTrainer as VALLE_V2_AR
+from models.tts.valle_v2.valle_nar_trainer import ValleNARTrainer as VALLE_V2_NAR
+from models.tts.jets.jets_trainer import JetsTrainer
+
+from utils.util import load_config
+
+
+def build_trainer(args, cfg):
+    supported_trainer = {
+        "FastSpeech2": FastSpeech2Trainer,
+        "VITS": VITSTrainer,
+        "VALLE": VALLETrainer,
+        "NaturalSpeech2": NS2Trainer,
+        "VALLE_V2_AR": VALLE_V2_AR,
+        "VALLE_V2_NAR": VALLE_V2_NAR,
+        "Jets": JetsTrainer,
+    }
+
+    trainer_class = supported_trainer[cfg.model_type]
+    trainer = trainer_class(args, cfg)
+    return trainer
+
+
+def cuda_relevant(deterministic=False):
+    torch.cuda.empty_cache()
+    # TF32 on Ampere and above
+    torch.backends.cuda.matmul.allow_tf32 = True
+    torch.backends.cudnn.enabled = True
+    torch.backends.cudnn.benchmark = False
+    torch.backends.cudnn.allow_tf32 = True
+    # Deterministic
+    torch.backends.cudnn.deterministic = deterministic
+    torch.backends.cudnn.benchmark = not deterministic
+    torch.use_deterministic_algorithms(deterministic)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config",
+        default="config.json",
+        help="json files for configurations.",
+        required=True,
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=1234,
+        help="random seed",
+        required=False,
+    )
+    parser.add_argument(
+        "--exp_name",
+        type=str,
+        default="exp_name",
+        help="A specific name to note the experiment",
+        required=True,
+    )
+    parser.add_argument(
+        "--resume", action="store_true", help="The model name to restore"
+    )
+    parser.add_argument(
+        "--test", action="store_true", default=False, help="Test the model"
+    )
+    parser.add_argument(
+        "--log_level", default="warning", help="logging level (debug, info, warning)"
+    )
+    parser.add_argument(
+        "--resume_type",
+        type=str,
+        default="resume",
+        help="Resume training or finetuning.",
+    )
+    parser.add_argument(
+        "--checkpoint_path",
+        type=str,
+        default=None,
+        help="Checkpoint for resume training or finetuning.",
+    )
+    parser.add_argument(
+        "--resume_from_ckpt_path",
+        type=str,
+        default="",
+        help="Checkpoint for resume training or finetuning.",
+    )
+    # VALLETrainer.add_arguments(parser)
+    args = parser.parse_args()
+    cfg = load_config(args.config)
+
+    # Data Augmentation
+    if hasattr(cfg, "preprocess"):
+        if hasattr(cfg.preprocess, "data_augment"):
+            if (
+                type(cfg.preprocess.data_augment) == list
+                and len(cfg.preprocess.data_augment) > 0
+            ):
+                new_datasets_list = []
+                for dataset in cfg.preprocess.data_augment:
+                    new_datasets = [
+                        (
+                            f"{dataset}_pitch_shift"
+                            if cfg.preprocess.use_pitch_shift
+                            else None
+                        ),
+                        (
+                            f"{dataset}_formant_shift"
+                            if cfg.preprocess.use_formant_shift
+                            else None
+                        ),
+                        (
+                            f"{dataset}_equalizer"
+                            if cfg.preprocess.use_equalizer
+                            else None
+                        ),
+                        (
+                            f"{dataset}_time_stretch"
+                            if cfg.preprocess.use_time_stretch
+                            else None
+                        ),
+                    ]
+                    new_datasets_list.extend(filter(None, new_datasets))
+                cfg.dataset.extend(new_datasets_list)
+
+    print("experiment name: ", args.exp_name)
+    # # CUDA settings
+    cuda_relevant()
+
+    # Build trainer
+    print(f"Building {cfg.model_type} trainer")
+    trainer = build_trainer(args, cfg)
+    print(f"Start training {cfg.model_type} model")
+    if args.test:
+        trainer.test_loop()
+    else:
+        trainer.train_loop()
+
+
+if __name__ == "__main__":
+    main()

bins/vocoder/inference.py

@@ -0,0 +1,115 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+import os
+
+import torch
+
+from models.vocoders.vocoder_inference import VocoderInference
+from utils.util import load_config
+
+
+def build_inference(args, cfg, infer_type="infer_from_dataset"):
+    supported_inference = {
+        "GANVocoder": VocoderInference,
+        "DiffusionVocoder": VocoderInference,
+    }
+
+    inference_class = supported_inference[cfg.model_type]
+    return inference_class(args, cfg, infer_type)
+
+
+def cuda_relevant(deterministic=False):
+    torch.cuda.empty_cache()
+    # TF32 on Ampere and above
+    torch.backends.cuda.matmul.allow_tf32 = True
+    torch.backends.cudnn.enabled = True
+    torch.backends.cudnn.allow_tf32 = True
+    # Deterministic
+    torch.backends.cudnn.deterministic = deterministic
+    torch.backends.cudnn.benchmark = not deterministic
+    torch.use_deterministic_algorithms(deterministic)
+
+
+def build_parser():
+    r"""Build argument parser for inference.py.
+    Anything else should be put in an extra config YAML file.
+    """
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config",
+        type=str,
+        required=True,
+        help="JSON/YAML file for configurations.",
+    )
+    parser.add_argument(
+        "--infer_mode",
+        type=str,
+        required=None,
+    )
+    parser.add_argument(
+        "--infer_datasets",
+        nargs="+",
+        default=None,
+    )
+    parser.add_argument(
+        "--feature_folder",
+        type=str,
+        default=None,
+    )
+    parser.add_argument(
+        "--audio_folder",
+        type=str,
+        default=None,
+    )
+    parser.add_argument(
+        "--vocoder_dir",
+        type=str,
+        required=True,
+        help="Vocoder checkpoint directory. Searching behavior is the same as "
+        "the acoustics one.",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="result",
+        help="Output directory. Default: ./result",
+    )
+    parser.add_argument(
+        "--log_level",
+        type=str,
+        default="warning",
+        help="Logging level. Default: warning",
+    )
+    parser.add_argument(
+        "--keep_cache",
+        action="store_true",
+        default=False,
+        help="Keep cache files. Only applicable to inference from files.",
+    )
+    return parser
+
+
+def main():
+    # Parse arguments
+    args = build_parser().parse_args()
+
+    # Parse config
+    cfg = load_config(args.config)
+
+    # CUDA settings
+    cuda_relevant()
+
+    # Build inference
+    trainer = build_inference(args, cfg, args.infer_mode)
+
+    # Run inference
+    trainer.inference()
+
+
+if __name__ == "__main__":
+    main()

bins/vocoder/preprocess.py

@@ -0,0 +1,151 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import faulthandler
+
+faulthandler.enable()
+
+import os
+import argparse
+import json
+import pyworld as pw
+from multiprocessing import cpu_count
+
+
+from utils.util import load_config
+from preprocessors.processor import preprocess_dataset, prepare_align
+from preprocessors.metadata import cal_metadata
+from processors import acoustic_extractor, content_extractor, data_augment
+
+
+def extract_acoustic_features(dataset, output_path, cfg, n_workers=1):
+    """Extract acoustic features of utterances in the dataset
+
+    Args:
+        dataset (str): name of dataset, e.g. opencpop
+        output_path (str): directory that stores train, test and feature files of datasets
+        cfg (dict): dictionary that stores configurations
+        n_workers (int, optional): num of processes to extract features in parallel. Defaults to 1.
+    """
+    types = ["train", "test"] if "eval" not in dataset else ["test"]
+    metadata = []
+    for dataset_type in types:
+        dataset_output = os.path.join(output_path, dataset)
+        dataset_file = os.path.join(dataset_output, "{}.json".format(dataset_type))
+        with open(dataset_file, "r") as f:
+            metadata.extend(json.load(f))
+
+    acoustic_extractor.extract_utt_acoustic_features_serial(
+        metadata, dataset_output, cfg
+    )
+
+
+def preprocess(cfg, args):
+    """Proprocess raw data of single or multiple datasets (in cfg.dataset)
+
+    Args:
+        cfg (dict): dictionary that stores configurations
+        args (ArgumentParser): specify the configuration file and num_workers
+    """
+    # Specify the output root path to save the processed data
+    output_path = cfg.preprocess.processed_dir
+    os.makedirs(output_path, exist_ok=True)
+
+    ## Split train and test sets
+    for dataset in cfg.dataset:
+        print("Preprocess {}...".format(dataset))
+
+        preprocess_dataset(
+            dataset,
+            cfg.dataset_path[dataset],
+            output_path,
+            cfg.preprocess,
+            cfg.task_type,
+            is_custom_dataset=dataset in cfg.use_custom_dataset,
+        )
+
+    # Data augmentation: create new wav files with pitch shift, formant shift, equalizer, time stretch
+    try:
+        assert isinstance(
+            cfg.preprocess.data_augment, list
+        ), "Please provide a list of datasets need to be augmented."
+        if len(cfg.preprocess.data_augment) > 0:
+            new_datasets_list = []
+            for dataset in cfg.preprocess.data_augment:
+                new_datasets = data_augment.augment_dataset(cfg, dataset)
+                new_datasets_list.extend(new_datasets)
+            cfg.dataset.extend(new_datasets_list)
+            print("Augmentation datasets: ", cfg.dataset)
+    except:
+        print("No Data Augmentation.")
+
+    # Dump metadata of datasets (singers, train/test durations, etc.)
+    cal_metadata(cfg)
+
+    ## Prepare the acoustic features
+    for dataset in cfg.dataset:
+        # Skip augmented datasets which do not need to extract acoustic features
+        # We will copy acoustic features from the original dataset later
+        if (
+            "pitch_shift" in dataset
+            or "formant_shift" in dataset
+            or "equalizer" in dataset in dataset
+        ):
+            continue
+        print(
+            "Extracting acoustic features for {} using {} workers ...".format(
+                dataset, args.num_workers
+            )
+        )
+        extract_acoustic_features(dataset, output_path, cfg, args.num_workers)
+        # Calculate the statistics of acoustic features
+        if cfg.preprocess.mel_min_max_norm:
+            acoustic_extractor.cal_mel_min_max(dataset, output_path, cfg)
+
+    # Copy acoustic features for augmented datasets by creating soft-links
+    for dataset in cfg.dataset:
+        if "pitch_shift" in dataset:
+            src_dataset = dataset.replace("_pitch_shift", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        elif "formant_shift" in dataset:
+            src_dataset = dataset.replace("_formant_shift", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        elif "equalizer" in dataset:
+            src_dataset = dataset.replace("_equalizer", "")
+            src_dataset_dir = os.path.join(output_path, src_dataset)
+        else:
+            continue
+        dataset_dir = os.path.join(output_path, dataset)
+        metadata = []
+        for split in ["train", "test"] if not "eval" in dataset else ["test"]:
+            metadata_file_path = os.path.join(src_dataset_dir, "{}.json".format(split))
+            with open(metadata_file_path, "r") as f:
+                metadata.extend(json.load(f))
+        print("Copying acoustic features for {}...".format(dataset))
+        acoustic_extractor.copy_acoustic_features(
+            metadata, dataset_dir, src_dataset_dir, cfg
+        )
+        if cfg.preprocess.mel_min_max_norm:
+            acoustic_extractor.cal_mel_min_max(dataset, output_path, cfg)
+
+        if cfg.preprocess.extract_pitch:
+            acoustic_extractor.cal_pitch_statistics(dataset, output_path, cfg)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config", default="config.json", help="json files for configurations."
+    )
+    parser.add_argument("--num_workers", type=int, default=int(cpu_count()))
+
+    args = parser.parse_args()
+    cfg = load_config(args.config)
+
+    preprocess(cfg, args)
+
+
+if __name__ == "__main__":
+    main()

bins/vocoder/train.py

@@ -0,0 +1,93 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+
+import torch
+
+from models.vocoders.gan.gan_vocoder_trainer import GANVocoderTrainer
+from models.vocoders.diffusion.diffusion_vocoder_trainer import DiffusionVocoderTrainer
+
+from utils.util import load_config
+
+
+def build_trainer(args, cfg):
+    supported_trainer = {
+        "GANVocoder": GANVocoderTrainer,
+        "DiffusionVocoder": DiffusionVocoderTrainer,
+    }
+
+    trainer_class = supported_trainer[cfg.model_type]
+    trainer = trainer_class(args, cfg)
+    return trainer
+
+
+def cuda_relevant(deterministic=False):
+    torch.cuda.empty_cache()
+    # TF32 on Ampere and above
+    torch.backends.cuda.matmul.allow_tf32 = True
+    torch.backends.cudnn.enabled = True
+    torch.backends.cudnn.allow_tf32 = True
+    # Deterministic
+    torch.backends.cudnn.deterministic = deterministic
+    torch.backends.cudnn.benchmark = not deterministic
+    torch.use_deterministic_algorithms(deterministic)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config",
+        default="config.json",
+        help="json files for configurations.",
+        required=True,
+    )
+    parser.add_argument(
+        "--exp_name",
+        type=str,
+        default="exp_name",
+        help="A specific name to note the experiment",
+        required=True,
+    )
+    parser.add_argument(
+        "--resume_type",
+        type=str,
+        help="resume for continue to train, finetune for finetuning",
+    )
+    parser.add_argument(
+        "--checkpoint",
+        type=str,
+        help="checkpoint to resume",
+    )
+    parser.add_argument(
+        "--log_level", default="warning", help="logging level (debug, info, warning)"
+    )
+    args = parser.parse_args()
+    cfg = load_config(args.config)
+
+    # Data Augmentation
+    if cfg.preprocess.data_augment:
+        new_datasets_list = []
+        for dataset in cfg.preprocess.data_augment:
+            new_datasets = [
+                # f"{dataset}_pitch_shift",
+                # f"{dataset}_formant_shift",
+                f"{dataset}_equalizer",
+                f"{dataset}_time_stretch",
+            ]
+            new_datasets_list.extend(new_datasets)
+        cfg.dataset.extend(new_datasets_list)
+
+    # CUDA settings
+    cuda_relevant()
+
+    # Build trainer
+    trainer = build_trainer(args, cfg)
+
+    trainer.train_loop()
+
+
+if __name__ == "__main__":
+    main()

config/audioldm.json

@@ -0,0 +1,92 @@
+{
+  "base_config": "config/base.json",
+  "model_type": "AudioLDM",
+  "task_type": "tta",
+  "dataset": [
+    "AudioCaps"
+  ],
+  "preprocess": {
+    // feature used for model training
+    "use_spkid": false,
+    "use_uv": false,
+    "use_frame_pitch": false,
+    "use_phone_pitch": false,
+    "use_frame_energy": false,
+    "use_phone_energy": false,
+    "use_mel": false,
+    "use_audio": false,
+    "use_label": false,
+    "use_one_hot": false,
+    "cond_mask_prob": 0.1
+  },
+  // model
+  "model": {
+    "audioldm": {
+      "image_size": 32,
+      "in_channels": 4,
+      "out_channels": 4,
+      "model_channels": 256,
+      "attention_resolutions": [
+        4,
+        2,
+        1
+      ],
+      "num_res_blocks": 2,
+      "channel_mult": [
+        1,
+        2,
+        4
+      ],
+      "num_heads": 8,
+      "use_spatial_transformer": true,
+      "transformer_depth": 1,
+      "context_dim": 768,
+      "use_checkpoint": true,
+      "legacy": false
+    },
+    "autoencoderkl": {
+      "ch": 128,
+      "ch_mult": [
+        1,
+        1,
+        2,
+        2,
+        4
+      ],
+      "num_res_blocks": 2,
+      "in_channels": 1,
+      "z_channels": 4,
+      "out_ch": 1,
+      "double_z": true
+    },
+    "noise_scheduler": {
+      "num_train_timesteps": 1000,
+      "beta_start": 0.00085,
+      "beta_end": 0.012,
+      "beta_schedule": "scaled_linear",
+      "clip_sample": false,
+      "steps_offset": 1,
+      "set_alpha_to_one": false,
+      "skip_prk_steps": true,
+      "prediction_type": "epsilon"
+    }
+  },
+  // train
+  "train": {
+    "lronPlateau": {
+      "factor": 0.9,
+      "patience": 100,
+      "min_lr": 4.0e-5,
+      "verbose": true
+    },
+    "adam": {
+      "lr": 5.0e-5,
+      "betas": [
+        0.9,
+        0.999
+      ],
+      "weight_decay": 1.0e-2,
+      "eps": 1.0e-8
+    }
+  }
+}

config/autoencoderkl.json

@@ -0,0 +1,69 @@
+{
+  "base_config": "config/base.json",
+  "model_type": "AutoencoderKL",
+  "task_type": "tta",
+  "dataset": [
+    "AudioCaps"
+  ],
+  "preprocess": {
+    // feature used for model training
+    "use_spkid": false,
+    "use_uv": false,
+    "use_frame_pitch": false,
+    "use_phone_pitch": false,
+    "use_frame_energy": false,
+    "use_phone_energy": false,
+    "use_mel": false,
+    "use_audio": false,
+    "use_label": false,
+    "use_one_hot": false
+  },
+  // model
+  "model": {
+    "autoencoderkl": {
+      "ch": 128,
+      "ch_mult": [
+        1,
+        1,
+        2,
+        2,
+        4
+      ],
+      "num_res_blocks": 2,
+      "in_channels": 1,
+      "z_channels": 4,
+      "out_ch": 1,
+      "double_z": true
+    },
+    "loss": {
+      "kl_weight": 1e-8,
+      "disc_weight": 0.5,
+      "disc_factor": 1.0,
+      "logvar_init": 0.0,
+      "min_adapt_d_weight": 0.0,
+      "max_adapt_d_weight": 10.0,
+      "disc_start": 50001,
+      "disc_in_channels": 1,
+      "disc_num_layers": 3,
+      "use_actnorm": false
+    }
+  },
+  // train
+  "train": {
+    "lronPlateau": {
+      "factor": 0.9,
+      "patience": 100,
+      "min_lr": 4.0e-5,
+      "verbose": true
+    },
+    "adam": {
+      "lr": 4.0e-4,
+      "betas": [
+        0.9,
+        0.999
+      ],
+      "weight_decay": 1.0e-2,
+      "eps": 1.0e-8
+    }
+  }
+}

config/base.json

@@ -0,0 +1,185 @@
+{
+  "supported_model_type": [
+    "GANVocoder",
+    "Fastspeech2",
+    "DiffSVC",
+    "Transformer",
+    "EDM",
+    "CD"
+  ],
+  "task_type": "",
+  "dataset": [],
+  "use_custom_dataset": [],
+  "preprocess": {
+    "phone_extractor": "espeak", // "espeak, pypinyin, pypinyin_initials_finals, lexicon"
+    // trim audio silence
+    "data_augment": false,
+    "trim_silence": false,
+    "num_silent_frames": 8,
+    "trim_fft_size": 512, // fft size used in trimming
+    "trim_hop_size": 128, // hop size used in trimming
+    "trim_top_db": 30, // top db used in trimming sensitive to each dataset
+    // acoustic features
+    "extract_mel": false,
+    "mel_extract_mode": "",
+    "extract_linear_spec": false,
+    "extract_mcep": false,
+    "extract_pitch": false,
+    "extract_acoustic_token": false,
+    "pitch_remove_outlier": false,
+    "extract_uv": false,
+    "pitch_norm": false,
+    "extract_audio": false,
+    "extract_label": false,
+    "pitch_extractor": "parselmouth", // pyin, dio, pyworld, pyreaper, parselmouth, CWT (Continuous Wavelet Transform)
+    "extract_energy": false,
+    "energy_remove_outlier": false,
+    "energy_norm": false,
+    "energy_extract_mode": "from_mel",
+    "extract_duration": false,
+    "extract_amplitude_phase": false,
+    "mel_min_max_norm": false,
+    // lingusitic features
+    "extract_phone": false,
+    "lexicon_path": "./text/lexicon/librispeech-lexicon.txt",
+    // content features
+    "extract_whisper_feature": false,
+    "extract_contentvec_feature": false,
+    "extract_mert_feature": false,
+    "extract_wenet_feature": false,
+    // Settings for data preprocessing
+    "n_mel": 80,
+    "win_size": 480,
+    "hop_size": 120,
+    "sample_rate": 24000,
+    "n_fft": 1024,
+    "fmin": 0,
+    "fmax": 12000,
+    "min_level_db": -115,
+    "ref_level_db": 20,
+    "bits": 8,
+    // Directory names of processed data or extracted features
+    "processed_dir": "processed_data",
+    "trimmed_wav_dir": "trimmed_wavs", // directory name of silence trimed wav
+    "raw_data": "raw_data",
+    "phone_dir": "phones",
+    "wav_dir": "wavs", // directory name of processed wav (such as downsampled waveform)
+    "audio_dir": "audios",
+    "log_amplitude_dir": "log_amplitudes",
+    "phase_dir": "phases",
+    "real_dir": "reals",
+    "imaginary_dir": "imaginarys",
+    "label_dir": "labels",
+    "linear_dir": "linears",
+    "mel_dir": "mels", // directory name of extraced mel features
+    "mcep_dir": "mcep", // directory name of extraced mcep features
+    "dur_dir": "durs",
+    "symbols_dict": "symbols.dict",
+    "lab_dir": "labs", // directory name of extraced label features
+    "wenet_dir": "wenet", // directory name of extraced wenet features
+    "contentvec_dir": "contentvec", // directory name of extraced wenet features
+    "pitch_dir": "pitches", // directory name of extraced pitch features
+    "energy_dir": "energys", // directory name of extracted energy features
+    "phone_pitch_dir": "phone_pitches", // directory name of extraced pitch features
+    "phone_energy_dir": "phone_energys", // directory name of extracted energy features
+    "uv_dir": "uvs", // directory name of extracted unvoiced features
+    "duration_dir": "duration", // ground-truth duration file
+    "phone_seq_file": "phone_seq_file", // phoneme sequence file
+    "file_lst": "file.lst",
+    "train_file": "train.json", // training set, the json file contains detailed information about the dataset, including dataset name, utterance id, duration of the utterance
+    "valid_file": "valid.json", // validattion set
+    "spk2id": "spk2id.json", // used for multi-speaker dataset
+    "utt2spk": "utt2spk", // used for multi-speaker dataset
+    "emo2id": "emo2id.json", // used for multi-emotion dataset
+    "utt2emo": "utt2emo", // used for multi-emotion dataset
+    // Features used for model training
+    "use_text": false,
+    "use_phone": false,
+    "use_phn_seq": false,
+    "use_lab": false,
+    "use_linear": false,
+    "use_mel": false,
+    "use_min_max_norm_mel": false,
+    "use_wav": false,
+    "use_phone_pitch": false,
+    "use_log_scale_pitch": false,
+    "use_phone_energy": false,
+    "use_phone_duration": false,
+    "use_log_scale_energy": false,
+    "use_wenet": false,
+    "use_dur": false,
+    "use_spkid": false, // True: use speaker id for multi-speaker dataset
+    "use_emoid": false, // True: use emotion id for multi-emotion dataset
+    "use_frame_pitch": false,
+    "use_uv": false,
+    "use_frame_energy": false,
+    "use_frame_duration": false,
+    "use_audio": false,
+    "use_label": false,
+    "use_one_hot": false,
+    "use_amplitude_phase": false,
+    "align_mel_duration": false
+  },
+  "train": {
+    "ddp": true,
+    "batch_size": 16,
+    "max_steps": 1000000,
+    // Trackers
+    "tracker": [
+      "tensorboard"
+      // "wandb",
+      // "cometml",
+      // "mlflow",
+    ],
+    "max_epoch": -1,
+    // -1 means no limit
+    "save_checkpoint_stride": [
+      5,
+      20
+    ],
+    // unit is epoch
+    "keep_last": [
+      3,
+      -1
+    ],
+    // -1 means infinite, if one number will broadcast
+    "run_eval": [
+      false,
+      true
+    ],
+    // if one number will broadcast
+    // Fix the random seed
+    "random_seed": 10086,
+    // Optimizer
+    "optimizer": "AdamW",
+    "adamw": {
+      "lr": 4.0e-4
+      // nn model lr
+    },
+    // LR Scheduler
+    "scheduler": "ReduceLROnPlateau",
+    "reducelronplateau": {
+      "factor": 0.8,
+      "patience": 10,
+      // unit is epoch
+      "min_lr": 1.0e-4
+    },
+    // Batchsampler
+    "sampler": {
+      "holistic_shuffle": true,
+      "drop_last": true
+    },
+    // Dataloader
+    "dataloader": {
+      "num_worker": 32,
+      "pin_memory": true
+    },
+    "gradient_accumulation_step": 1,
+    "total_training_steps": 50000,
+    "save_summary_steps": 500,
+    "save_checkpoints_steps": 10000,
+    "valid_interval": 10000,
+    "keep_checkpoint_max": 5,
+    "multi_speaker_training": false // True: train multi-speaker model; False: training single-speaker model;
+  }
+}

config/comosvc.json

@@ -0,0 +1,215 @@
+{
+    "base_config": "config/svc/base.json",
+    "model_type": "DiffComoSVC",
+    "task_type": "svc",
+    "preprocess": {
+        // data augmentations
+        "use_pitch_shift": false,
+        "use_formant_shift": false,
+        "use_time_stretch": false,
+        "use_equalizer": false,
+        // acoustic features
+        "extract_mel": true,
+        "mel_min_max_norm": true,
+        "extract_pitch": true,
+        "pitch_extractor": "parselmouth",
+        "extract_uv": true,
+        "extract_energy": true,
+        // content features
+        "extract_whisper_feature": false,
+        "whisper_sample_rate": 16000,
+        "extract_contentvec_feature": false,
+        "contentvec_sample_rate": 16000,
+        "extract_wenet_feature": false,
+        "wenet_sample_rate": 16000,
+        "extract_mert_feature": false,
+        "mert_sample_rate": 16000,
+        // Default config for whisper
+        "whisper_frameshift": 0.01,
+        "whisper_downsample_rate": 2,
+        // Default config for content vector
+        "contentvec_frameshift": 0.02,
+        // Default config for mert
+        "mert_model": "m-a-p/MERT-v1-330M",
+        "mert_feature_layer": -1,
+        "mert_hop_size": 320,
+        // 24k
+        "mert_frameshit": 0.01333,
+        // 10ms
+        "wenet_frameshift": 0.01,
+        // wenetspeech is 4, gigaspeech is 6
+        "wenet_downsample_rate": 4,
+        // Default config
+        "n_mel": 100,
+        "win_size": 1024,
+        // todo
+        "hop_size": 256,
+        "sample_rate": 24000,
+        "n_fft": 1024,
+        // todo
+        "fmin": 0,
+        "fmax": 12000,
+        // todo
+        "f0_min": 50,
+        // ~C2
+        "f0_max": 1100,
+        //1100,    // ~C6(1100), ~G5(800)
+        "pitch_bin": 256,
+        "pitch_max": 1100.0,
+        "pitch_min": 50.0,
+        "is_label": true,
+        "is_mu_law": true,
+        "bits": 8,
+        "mel_min_max_stats_dir": "mel_min_max_stats",
+        "whisper_dir": "whisper",
+        "contentvec_dir": "contentvec",
+        "wenet_dir": "wenet",
+        "mert_dir": "mert",
+        // Extract content features using dataloader
+        "pin_memory": true,
+        "num_workers": 8,
+        "content_feature_batch_size": 16,
+        // Features used for model training
+        "use_mel": true,
+        "use_min_max_norm_mel": true,
+        "use_frame_pitch": true,
+        "use_uv": true,
+        "use_frame_energy": true,
+        "use_log_scale_pitch": false,
+        "use_log_scale_energy": false,
+        "use_spkid": true,
+        // Meta file
+        "train_file": "train.json",
+        "valid_file": "test.json",
+        "spk2id": "singers.json",
+        "utt2spk": "utt2singer"
+    },
+    "model": {
+        "teacher_model_path": "[Your Teacher Model Path].bin",
+        "condition_encoder": {
+            "merge_mode": "add",
+            "input_melody_dim": 1,
+            "use_log_f0": true,
+            "n_bins_melody": 256,
+            //# Quantization (0 for not quantization)
+            "output_melody_dim": 384,
+            "input_loudness_dim": 1,
+            "use_log_loudness": true,
+            "n_bins_loudness": 256,
+            "output_loudness_dim": 384,
+            "use_whisper": false,
+            "use_contentvec": false,
+            "use_wenet": false,
+            "use_mert": false,
+            "whisper_dim": 1024,
+            "contentvec_dim": 256,
+            "mert_dim": 256,
+            "wenet_dim": 512,
+            "content_encoder_dim": 384,
+            "output_singer_dim": 384,
+            "singer_table_size": 512,
+            "output_content_dim": 384,
+            "use_spkid": true
+        },
+        "comosvc": {
+            "distill": false,
+            // conformer encoder
+            "input_dim": 384,
+            "output_dim": 100,
+            "n_heads": 2,
+            "n_layers": 6,
+            "filter_channels": 512,
+            "dropout": 0.1,
+            // karras diffusion
+            "P_mean": -1.2,
+            "P_std": 1.2,
+            "sigma_data": 0.5,
+            "sigma_min": 0.002,
+            "sigma_max": 80,
+            "rho": 7,
+            "n_timesteps": 18,
+        },
+        "diffusion": {
+            // Diffusion steps encoder
+            "step_encoder": {
+                "dim_raw_embedding": 128,
+                "dim_hidden_layer": 512,
+                "activation": "SiLU",
+                "num_layer": 2,
+                "max_period": 10000
+            },
+            // Diffusion decoder
+            "model_type": "bidilconv",
+            // bidilconv, unet2d, TODO: unet1d
+            "bidilconv": {
+                "base_channel": 384,
+                "n_res_block": 20,
+                "conv_kernel_size": 3,
+                "dilation_cycle_length": 4,
+                // specially, 1 means no dilation
+                "conditioner_size": 100
+            }
+        },
+    },
+    "train": {
+        // Basic settings
+        "fast_steps": 0,
+        "batch_size": 64,
+        "gradient_accumulation_step": 1,
+        "max_epoch": -1,
+        // -1 means no limit
+        "save_checkpoint_stride": [
+            10,
+            100
+        ],
+        // unit is epoch
+        "keep_last": [
+            3,
+            -1
+        ],
+        // -1 means infinite, if one number will broadcast
+        "run_eval": [
+            false,
+            true
+        ],
+        // if one number will broadcast
+        // Fix the random seed
+        "random_seed": 10086,
+        // Batchsampler
+        "sampler": {
+            "holistic_shuffle": true,
+            "drop_last": true
+        },
+        // Dataloader
+        "dataloader": {
+            "num_worker": 32,
+            "pin_memory": true
+        },
+        // Trackers
+        "tracker": [
+            "tensorboard"
+            // "wandb",
+            // "cometml",
+            // "mlflow",
+        ],
+        // Optimizer
+        "optimizer": "AdamW",
+        "adamw": {
+            "lr": 5.0e-5
+            // nn model lr
+        },
+        // LR Scheduler
+        "scheduler": "ReduceLROnPlateau",
+        "reducelronplateau": {
+            "factor": 0.8,
+            "patience": 10,
+            // unit is epoch
+            "min_lr": 5.0e-6
+        }
+    },
+    "inference": {
+        "comosvc": {
+            "inference_steps": 40
+        }
+    }
+}

config/fs2.json

@@ -0,0 +1,120 @@
+{
+    "base_config": "config/tts.json",
+    "model_type": "FastSpeech2",
+    "task_type": "tts",
+    "dataset": ["LJSpeech"], 
+    "preprocess": {
+      // acoustic features
+      "extract_audio": true,
+      "extract_mel": true,
+      "mel_extract_mode": "taco",
+      "mel_min_max_norm": false,
+      "extract_pitch": true,
+      "extract_uv": false,
+      "pitch_extractor": "dio",
+      "extract_energy": true,
+      "energy_extract_mode": "from_tacotron_stft",
+      "extract_duration": true,
+      "use_phone": false,
+      "pitch_norm": true,
+      "energy_norm": true,
+      "pitch_remove_outlier": true,
+      "energy_remove_outlier": true,
+
+      // Default config 
+      "n_mel": 80,
+      "win_size": 1024,  // todo
+      "hop_size": 256,
+      "sample_rate": 22050,
+      "n_fft": 1024, // todo
+      "fmin": 0,
+      "fmax": 8000, // todo
+      "raw_data": "raw_data",
+      "text_cleaners": ["english_cleaners"],
+      "f0_min": 71,    // ~C2
+      "f0_max": 800, //1100,    // ~C6(1100), ~G5(800)
+      "pitch_bin": 256,
+      "pitch_max": 1100.0,
+      "pitch_min": 50.0,
+      "is_label": true,
+      "is_mu_law": true,
+      "bits": 8,
+
+      "mel_min_max_stats_dir": "mel_min_max_stats",
+      "whisper_dir": "whisper",
+      "content_vector_dir": "content_vector",
+      "wenet_dir": "wenet",
+      "mert_dir": "mert",
+      "spk2id":"spk2id.json",
+      "utt2spk":"utt2spk",
+      "valid_file": "test.json",
+
+      // Features used for model training
+      "use_mel": true,
+      "use_min_max_norm_mel": false,
+      "use_frame_pitch": false,
+      "use_frame_energy": false,
+      "use_phone_pitch": true,
+      "use_phone_energy": true,
+      "use_log_scale_pitch": false,
+      "use_log_scale_energy": false,
+      "use_spkid": false,
+      "align_mel_duration": true,
+      "text_cleaners": ["english_cleaners"],
+      "phone_extractor": "lexicon", // "espeak, pypinyin, pypinyin_initials_finals, lexicon (only for language=en-us right now)"
+      },
+    "model": {
+      // Settings for transformer
+      "transformer": {
+        "encoder_layer": 4,
+        "encoder_head": 2,
+        "encoder_hidden": 256,
+        "decoder_layer": 6,
+        "decoder_head": 2,
+        "decoder_hidden": 256,
+        "conv_filter_size": 1024,
+        "conv_kernel_size": [9, 1],
+        "encoder_dropout": 0.2,
+        "decoder_dropout": 0.2
+      },
+
+      // Settings for variance_predictor
+      "variance_predictor":{
+        "filter_size": 256,
+        "kernel_size": 3,
+        "dropout": 0.5
+      },
+    "variance_embedding":{
+        "pitch_quantization": "linear", // support 'linear' or 'log', 'log' is allowed only if the pitch values are not normalized during preprocessing
+        "energy_quantization": "linear", // support 'linear' or 'log', 'log' is allowed only if the energy values are not normalized during preprocessing
+        "n_bins": 256
+      },
+    "max_seq_len": 1000
+    },
+    "train":{
+      "batch_size": 16,
+      "max_epoch": 100,
+      "sort_sample": true,
+      "drop_last": true,
+      "group_size": 4,
+      "grad_clip_thresh": 1.0,
+      "dataloader": {
+        "num_worker": 8,
+        "pin_memory": true
+      },
+      "lr_scheduler":{
+        "num_warmup": 4000
+      },
+      // LR Scheduler
+      "scheduler": "NoamLR",
+      // Optimizer
+      "optimizer": "Adam",
+      "adam": {
+        "lr": 0.0625,
+        "betas": [0.9, 0.98],
+        "eps": 0.000000001,
+        "weight_decay": 0.0
+      },
+    }
+
+}

config/jets.json

@@ -0,0 +1,120 @@
+{
+    "base_config": "config/tts.json",
+    "model_type": "Jets",
+    "task_type": "tts",
+    "dataset": ["LJSpeech"], 
+    "preprocess": {
+      // acoustic features
+      "extract_audio": true,
+      "extract_mel": true,
+      "mel_extract_mode": "taco",
+      "mel_min_max_norm": false,
+      "extract_pitch": true,
+      "extract_uv": false,
+      "pitch_extractor": "dio",
+      "extract_energy": true,
+      "energy_extract_mode": "from_tacotron_stft",
+      "extract_duration": true,
+      "use_phone": false,
+      "pitch_norm": true,
+      "energy_norm": true,
+      "pitch_remove_outlier": true,
+      "energy_remove_outlier": true,
+
+      // Default config 
+      "n_mel": 80,
+      "win_size": 1024,  // todo
+      "hop_size": 256,
+      "sample_rate": 22050,
+      "n_fft": 1024, // todo
+      "fmin": 0,
+      "fmax": 8000, // todo
+      "raw_data": "raw_data",
+      "text_cleaners": ["english_cleaners"],
+      "f0_min": 71,    // ~C2
+      "f0_max": 800, //1100,    // ~C6(1100), ~G5(800)
+      "pitch_bin": 256,
+      "pitch_max": 1100.0,
+      "pitch_min": 50.0,
+      "is_label": true,
+      "is_mu_law": true,
+      "bits": 8,
+
+      "mel_min_max_stats_dir": "mel_min_max_stats",
+      "whisper_dir": "whisper",
+      "content_vector_dir": "content_vector",
+      "wenet_dir": "wenet",
+      "mert_dir": "mert",
+      "spk2id":"spk2id.json",
+      "utt2spk":"utt2spk",
+      "valid_file": "test.json",
+
+      // Features used for model training
+      "use_mel": true,
+      "use_min_max_norm_mel": false,
+      "use_frame_pitch": true,
+      "use_frame_energy": true,
+      "use_phone_pitch": false,
+      "use_phone_energy": false,
+      "use_log_scale_pitch": false,
+      "use_log_scale_energy": false,
+      "use_spkid": false,
+      "align_mel_duration": true,
+      "text_cleaners": ["english_cleaners"],
+      "phone_extractor": "lexicon", // "espeak, pypinyin, pypinyin_initials_finals, lexicon (only for language=en-us right now)"
+      },
+    "model": {
+      // Settings for transformer
+      "transformer": {
+        "encoder_layer": 4,
+        "encoder_head": 2,
+        "encoder_hidden": 256,
+        "decoder_layer": 6,
+        "decoder_head": 2,
+        "decoder_hidden": 256,
+        "conv_filter_size": 1024,
+        "conv_kernel_size": [9, 1],
+        "encoder_dropout": 0.2,
+        "decoder_dropout": 0.2
+      },
+
+      // Settings for variance_predictor
+      "variance_predictor":{
+        "filter_size": 256,
+        "kernel_size": 3,
+        "dropout": 0.5
+      },
+    "variance_embedding":{
+        "pitch_quantization": "linear", // support 'linear' or 'log', 'log' is allowed only if the pitch values are not normalized during preprocessing
+        "energy_quantization": "linear", // support 'linear' or 'log', 'log' is allowed only if the energy values are not normalized during preprocessing
+        "n_bins": 256
+      },
+    "max_seq_len": 1000
+    },
+    "train":{
+      "batch_size": 16,
+      "max_epoch": 100,
+      "sort_sample": true,
+      "drop_last": true,
+      "group_size": 4,
+      "grad_clip_thresh": 1.0,
+      "dataloader": {
+        "num_worker": 8,
+        "pin_memory": true
+      },
+      "lr_scheduler":{
+        "num_warmup": 4000
+      },
+      // LR Scheduler
+      "scheduler": "NoamLR",
+      // Optimizer
+      "optimizer": "Adam",
+      "adam": {
+        "lr": 0.0625,
+        "betas": [0.9, 0.98],
+        "eps": 0.000000001,
+        "weight_decay": 0.0
+      },
+    }
+
+}

config/ns2.json

@@ -0,0 +1,88 @@
+{
+    "base_config": "config/base.json",
+    "model_type": "NaturalSpeech2",
+    "dataset": ["libritts"],
+    "preprocess": {
+        "use_mel": false,
+        "use_code": true,
+        "use_spkid": true,
+        "use_pitch": true,
+        "use_duration": true,
+        "use_phone": true,
+        "use_len": true,
+        "use_cross_reference": true,
+        "train_file": "train.json",
+        "melspec_dir": "mel",
+        "code_dir": "code",
+        "pitch_dir": "pitch",
+        "duration_dir": "duration",
+        "clip_mode": "start"
+    },
+    "model": {
+        "latent_dim": 128,
+        "prior_encoder": {
+            "vocab_size": 100,
+            "pitch_min": 50,
+            "pitch_max": 1100,
+            "pitch_bins_num": 512,
+            "encoder": {
+                "encoder_layer": 6,
+                "encoder_hidden": 512,
+                "encoder_head": 8,
+                "conv_filter_size": 2048,
+                "conv_kernel_size": 9,
+                "encoder_dropout": 0.2,
+                "use_cln": true
+            },
+            "duration_predictor": {
+                "input_size": 512,
+                "filter_size": 512,
+                "kernel_size": 3,
+                "conv_layers": 30,
+                "cross_attn_per_layer": 3,
+                "attn_head": 8,
+                "drop_out": 0.5
+            },
+            "pitch_predictor": {
+                "input_size": 512,
+                "filter_size": 512,
+                "kernel_size": 5,
+                "conv_layers": 30,
+                "cross_attn_per_layer": 3,
+                "attn_head": 8,
+                "drop_out": 0.5
+            }
+        },
+        "diffusion": {
+            "wavenet": {
+                "input_size": 128,
+                "hidden_size": 512,
+                "out_size": 128,
+                "num_layers": 40,
+                "cross_attn_per_layer": 3,
+                "dilation_cycle": 2,
+                "attn_head": 8,
+                "drop_out": 0.2
+            },
+            "beta_min": 0.05,
+            "beta_max": 20,
+            "sigma": 1.0,
+            "noise_factor": 1.0,
+            "ode_solver": "euler"
+        },
+        "prompt_encoder": {
+            "encoder_layer": 6,
+            "encoder_hidden": 512,
+            "encoder_head": 8,
+            "conv_filter_size": 2048,
+            "conv_kernel_size": 9,
+            "encoder_dropout": 0.2,
+            "use_cln": false
+        },
+        "query_emb": {
+            "query_token_num": 32,
+            "hidden_size": 512,
+            "head_num": 8
+        }
+    }
+}

config/svc/base.json

@@ -0,0 +1,119 @@
+{
+    "base_config": "config/base.json",
+    "task_type": "svc",
+    "preprocess": {
+        // data augmentations
+        "use_pitch_shift": false,
+        "use_formant_shift": false,
+        "use_time_stretch": false,
+        "use_equalizer": false,
+        // Online or offline features extraction ("offline" or "online")
+        "features_extraction_mode": "offline",
+        // acoustic features
+        "extract_mel": true,
+        "mel_min_max_norm": true,
+        "extract_pitch": true,
+        "pitch_extractor": "parselmouth",
+        "extract_uv": true,
+        "extract_energy": true,
+        // content features
+        "extract_whisper_feature": false,
+        "whisper_sample_rate": 16000,
+        "extract_contentvec_feature": false,
+        "contentvec_sample_rate": 16000,
+        "extract_wenet_feature": false,
+        "wenet_sample_rate": 16000,
+        "extract_mert_feature": false,
+        "mert_sample_rate": 16000,
+        // Default config for whisper
+        "whisper_frameshift": 0.01,
+        "whisper_downsample_rate": 2,
+        // Default config for content vector
+        "contentvec_frameshift": 0.02,
+        // Default config for mert
+        "mert_model": "m-a-p/MERT-v1-330M",
+        "mert_feature_layer": -1,
+        "mert_hop_size": 320,
+        // 24k
+        "mert_frameshit": 0.01333,
+        // 10ms
+        "wenet_frameshift": 0.01,
+        // wenetspeech is 4, gigaspeech is 6
+        "wenet_downsample_rate": 4,
+        // Default config
+        "n_mel": 100,
+        "win_size": 1024,
+        // todo
+        "hop_size": 256,
+        "sample_rate": 24000,
+        "n_fft": 1024,
+        // todo
+        "fmin": 0,
+        "fmax": 12000,
+        // todo
+        "f0_min": 50,
+        // ~C2
+        "f0_max": 1100,
+        //1100,    // ~C6(1100), ~G5(800)
+        "pitch_bin": 256,
+        "pitch_max": 1100.0,
+        "pitch_min": 50.0,
+        "is_label": true,
+        "is_mu_law": true,
+        "bits": 8,
+        "mel_min_max_stats_dir": "mel_min_max_stats",
+        "whisper_dir": "whisper",
+        "contentvec_dir": "contentvec",
+        "wenet_dir": "wenet",
+        "mert_dir": "mert",
+        // Extract content features using dataloader
+        "pin_memory": true,
+        "num_workers": 8,
+        "content_feature_batch_size": 16,
+        // Features used for model training
+        "use_mel": true,
+        "use_min_max_norm_mel": true,
+        "use_frame_pitch": true,
+        "use_uv": true,
+        "use_interpolation_for_uv": false,
+        "use_frame_energy": true,
+        "use_log_scale_pitch": false,
+        "use_log_scale_energy": false,
+        "use_spkid": true,
+        // Meta file
+        "train_file": "train.json",
+        "valid_file": "test.json",
+        "spk2id": "singers.json",
+        "utt2spk": "utt2singer"
+    },
+    "model": {
+        "condition_encoder": {
+            "merge_mode": "add",
+            // Prosody Features
+            "use_f0": true,
+            "use_uv": true,
+            "use_energy": true,
+            // Quantization (0 for not quantization)
+            "input_melody_dim": 1,
+            "n_bins_melody": 256,
+            "output_melody_dim": 384,
+            "input_loudness_dim": 1,
+            "n_bins_loudness": 256,
+            "output_loudness_dim": 384,
+            // Semantic Features
+            "use_whisper": false,
+            "use_contentvec": false,
+            "use_wenet": false,
+            "use_mert": false,
+            "whisper_dim": 1024,
+            "contentvec_dim": 256,
+            "mert_dim": 256,
+            "wenet_dim": 512,
+            "content_encoder_dim": 384,
+            // Speaker Features
+            "output_singer_dim": 384,
+            "singer_table_size": 512,
+            "use_spkid": true
+        }
+    },
+}

config/svc/diffusion.json

@@ -0,0 +1,142 @@
+{
+    "base_config": "config/svc/base.json",
+    "model": {
+        "condition_encoder": {
+            "merge_mode": "add",
+            // Prosody Features
+            "use_f0": true,
+            "use_uv": true,
+            "use_energy": true,
+            // Quantization (0 for not quantization)
+            "input_melody_dim": 1,
+            "n_bins_melody": 256,
+            "output_melody_dim": 384,
+            "input_loudness_dim": 1,
+            "n_bins_loudness": 256,
+            "output_loudness_dim": 384,
+            // Semantic Features
+            "use_whisper": false,
+            "use_contentvec": false,
+            "use_wenet": false,
+            "use_mert": false,
+            "whisper_dim": 1024,
+            "contentvec_dim": 256,
+            "mert_dim": 256,
+            "wenet_dim": 512,
+            "content_encoder_dim": 384,
+            // Speaker Features
+            "output_singer_dim": 384,
+            "singer_table_size": 512,
+            "use_spkid": true
+        },
+        "diffusion": {
+            "scheduler": "ddpm",
+            "scheduler_settings": {
+                "num_train_timesteps": 1000,
+                "beta_start": 1.0e-4,
+                "beta_end": 0.02,
+                "beta_schedule": "linear"
+            },
+            // Diffusion steps encoder
+            "step_encoder": {
+                "dim_raw_embedding": 128,
+                "dim_hidden_layer": 512,
+                "activation": "SiLU",
+                "num_layer": 2,
+                "max_period": 10000
+            },
+            // Diffusion decoder
+            "model_type": "bidilconv",
+            // bidilconv, unet2d, TODO: unet1d
+            "bidilconv": {
+                "base_channel": 384,
+                "n_res_block": 20,
+                "conv_kernel_size": 3,
+                "dilation_cycle_length": 4,
+                // specially, 1 means no dilation
+                "conditioner_size": 384
+            },
+            "unet2d": {
+                "in_channels": 1,
+                "out_channels": 1,
+                "down_block_types": [
+                    "CrossAttnDownBlock2D",
+                    "CrossAttnDownBlock2D",
+                    "CrossAttnDownBlock2D",
+                    "DownBlock2D"
+                ],
+                "mid_block_type": "UNetMidBlock2DCrossAttn",
+                "up_block_types": [
+                    "UpBlock2D",
+                    "CrossAttnUpBlock2D",
+                    "CrossAttnUpBlock2D",
+                    "CrossAttnUpBlock2D"
+                ],
+                "only_cross_attention": false
+            }
+        }
+    },
+    "train": {
+        // Basic settings
+        "batch_size": 64,
+        "gradient_accumulation_step": 1,
+        "max_epoch": -1,
+        // -1 means no limit
+        "save_checkpoint_stride": [
+            5,
+            20
+        ],
+        // unit is epoch
+        "keep_last": [
+            3,
+            -1
+        ],
+        // -1 means infinite, if one number will broadcast
+        "run_eval": [
+            false,
+            true
+        ],
+        // if one number will broadcast
+        // Fix the random seed
+        "random_seed": 10086,
+        // Batchsampler
+        "sampler": {
+            "holistic_shuffle": true,
+            "drop_last": true
+        },
+        // Dataloader
+        "dataloader": {
+            "num_worker": 32,
+            "pin_memory": true
+        },
+        // Trackers
+        "tracker": [
+            "tensorboard"
+            // "wandb",
+            // "cometml",
+            // "mlflow",
+        ],
+        // Optimizer
+        "optimizer": "AdamW",
+        "adamw": {
+            "lr": 4.0e-4
+            // nn model lr
+        },
+        // LR Scheduler
+        "scheduler": "ReduceLROnPlateau",
+        "reducelronplateau": {
+            "factor": 0.8,
+            "patience": 10,
+            // unit is epoch
+            "min_lr": 1.0e-4
+        }
+    },
+    "inference": {
+        "diffusion": {
+            "scheduler": "pndm",
+            "scheduler_settings": {
+                "num_inference_timesteps": 1000
+            }
+        }
+    }
+}

config/transformer.json

@@ -0,0 +1,179 @@
+{
+    "base_config": "config/svc/base.json",
+    "model_type": "Transformer",
+    "task_type": "svc",
+    "preprocess": {
+        // data augmentations
+        "use_pitch_shift": false,
+        "use_formant_shift": false,
+        "use_time_stretch": false,
+        "use_equalizer": false,
+        // acoustic features
+        "extract_mel": true,
+        "mel_min_max_norm": true,
+        "extract_pitch": true,
+        "pitch_extractor": "parselmouth",
+        "extract_uv": true,
+        "extract_energy": true,
+        // content features
+        "extract_whisper_feature": false,
+        "whisper_sample_rate": 16000,
+        "extract_contentvec_feature": false,
+        "contentvec_sample_rate": 16000,
+        "extract_wenet_feature": false,
+        "wenet_sample_rate": 16000,
+        "extract_mert_feature": false,
+        "mert_sample_rate": 16000,
+        // Default config for whisper
+        "whisper_frameshift": 0.01,
+        "whisper_downsample_rate": 2,
+        // Default config for content vector
+        "contentvec_frameshift": 0.02,
+        // Default config for mert
+        "mert_model": "m-a-p/MERT-v1-330M",
+        "mert_feature_layer": -1,
+        "mert_hop_size": 320,
+        // 24k
+        "mert_frameshit": 0.01333,
+        // 10ms
+        "wenet_frameshift": 0.01,
+        // wenetspeech is 4, gigaspeech is 6
+        "wenet_downsample_rate": 4,
+        // Default config
+        "n_mel": 100,
+        "win_size": 1024,
+        // todo
+        "hop_size": 256,
+        "sample_rate": 24000,
+        "n_fft": 1024,
+        // todo
+        "fmin": 0,
+        "fmax": 12000,
+        // todo
+        "f0_min": 50,
+        // ~C2
+        "f0_max": 1100,
+        //1100,    // ~C6(1100), ~G5(800)
+        "pitch_bin": 256,
+        "pitch_max": 1100.0,
+        "pitch_min": 50.0,
+        "is_label": true,
+        "is_mu_law": true,
+        "bits": 8,
+        "mel_min_max_stats_dir": "mel_min_max_stats",
+        "whisper_dir": "whisper",
+        "contentvec_dir": "contentvec",
+        "wenet_dir": "wenet",
+        "mert_dir": "mert",
+        // Extract content features using dataloader
+        "pin_memory": true,
+        "num_workers": 8,
+        "content_feature_batch_size": 16,
+        // Features used for model training
+        "use_mel": true,
+        "use_min_max_norm_mel": true,
+        "use_frame_pitch": true,
+        "use_uv": true,
+        "use_frame_energy": true,
+        "use_log_scale_pitch": false,
+        "use_log_scale_energy": false,
+        "use_spkid": true,
+        // Meta file
+        "train_file": "train.json",
+        "valid_file": "test.json",
+        "spk2id": "singers.json",
+        "utt2spk": "utt2singer"
+    },
+    "model": {
+        "condition_encoder": {
+            "merge_mode": "add",
+            "input_melody_dim": 1,
+            "use_log_f0": true,
+            "n_bins_melody": 256,
+            //# Quantization (0 for not quantization)
+            "output_melody_dim": 384,
+            "input_loudness_dim": 1,
+            "use_log_loudness": true,
+            "n_bins_loudness": 256,
+            "output_loudness_dim": 384,
+            "use_whisper": false,
+            "use_contentvec": true,
+            "use_wenet": false,
+            "use_mert": false,
+            "whisper_dim": 1024,
+            "contentvec_dim": 256,
+            "mert_dim": 256,
+            "wenet_dim": 512,
+            "content_encoder_dim": 384,
+            "output_singer_dim": 384,
+            "singer_table_size": 512,
+            "output_content_dim": 384,
+            "use_spkid": true
+        },
+        "transformer": {
+            "type": "conformer",
+            // 'conformer' or 'transformer'
+            "input_dim": 384,
+            "output_dim": 100,
+            "n_heads": 2,
+            "n_layers": 6,
+            "filter_channels": 512,
+            "dropout": 0.1,
+        }
+    },
+    "train": {
+        // Basic settings
+        "batch_size": 64,
+        "gradient_accumulation_step": 1,
+        "max_epoch": -1,
+        // -1 means no limit
+        "save_checkpoint_stride": [
+            10,
+            100
+        ],
+        // unit is epoch
+        "keep_last": [
+            3,
+            -1
+        ],
+        // -1 means infinite, if one number will broadcast
+        "run_eval": [
+            false,
+            true
+        ],
+        // if one number will broadcast
+        // Fix the random seed
+        "random_seed": 10086,
+        // Batchsampler
+        "sampler": {
+            "holistic_shuffle": true,
+            "drop_last": true
+        },
+        // Dataloader
+        "dataloader": {
+            "num_worker": 32,
+            "pin_memory": true
+        },
+        // Trackers
+        "tracker": [
+            "tensorboard"
+            // "wandb",
+            // "cometml",
+            // "mlflow",
+        ],
+        // Optimizer
+        "optimizer": "AdamW",
+        "adamw": {
+            "lr": 4.0e-4
+            // nn model lr
+        },
+        // LR Scheduler
+        "scheduler": "ReduceLROnPlateau",
+        "reducelronplateau": {
+            "factor": 0.8,
+            "patience": 10,
+            // unit is epoch
+            "min_lr": 1.0e-4
+        }
+    }
+}

config/tts.json

@@ -0,0 +1,25 @@
+{
+  "base_config": "config/base.json",  
+  "supported_model_type": [
+    "Fastspeech2",
+    "VITS",
+    "VALLE",
+    "NaturalSpeech2"
+  ],
+  "task_type": "tts",
+  "preprocess": {
+    "language": "en-us", //  espeak supports 100 languages https://github.com/espeak-ng/espeak-ng/blob/master/docs/languages.md
+    // linguistic features
+    "extract_phone": true,
+    "phone_extractor": "espeak", // "espeak, pypinyin, pypinyin_initials_finals, lexicon (only for language=en-us right now)"
+    "lexicon_path": "./text/lexicon/librispeech-lexicon.txt",
+    // Directory names of processed data or extracted features
+    "phone_dir": "phones",
+    "use_phone": true,
+    "add_blank": true
+  },
+  "model": {
+      "text_token_num": 512,
+  }
+
+}

config/valle.json

@@ -0,0 +1,55 @@
+{
+    "base_config": "config/tts.json",
+    "model_type": "VALLE",
+    "task_type": "tts",
+    "dataset": [
+        "libritts"
+    ],
+    "preprocess": {
+        "extract_phone": true,
+        "phone_extractor": "espeak", // phoneme extractor: espeak, pypinyin, pypinyin_initials_finals or lexicon 
+        "extract_acoustic_token": true,
+        "acoustic_token_extractor": "Encodec", // acoustic token extractor: encodec, dac(todo)
+        "acoustic_token_dir": "acoutic_tokens",
+        "use_text": false,
+        "use_phone": true,
+        "use_acoustic_token": true,
+        "symbols_dict": "symbols.dict",
+        "min_duration": 0.5, // the duration lowerbound to filter the audio with duration < min_duration
+        "max_duration": 14, //  the duration uperbound to filter the audio with duration > max_duration.              
+        "sample_rate": 24000, 
+        "codec_hop_size": 320
+    },
+    "model": {
+        "text_token_num": 512,
+        "audio_token_num": 1024,
+        "decoder_dim": 1024, // embedding dimension of the decoder model
+        "nhead": 16, // number of attention heads in the decoder layers
+        "num_decoder_layers": 12, // number of decoder layers
+        "norm_first": true, // pre or post Normalization.
+        "add_prenet": false, // whether add PreNet after Inputs
+        "prefix_mode": 0, //  mode for how to prefix VALL-E NAR Decoder, 0: no prefix, 1: 0 to random, 2: random to random, 4: chunk of pre or post utterance
+        "share_embedding": true, // share the parameters of the output projection layer with the parameters of the acoustic embedding
+        "nar_scale_factor": 1, // model scale factor which will be assigned different meanings in different models
+        "prepend_bos": false, // whether prepend <BOS> to the acoustic tokens -> AR Decoder inputs
+        "num_quantizers": 8, // numbert of the audio quantization layers
+        // "scaling_xformers": false, // Apply Reworked Conformer scaling on Transformers 
+    },
+    "train": {
+        "use_dynamic_batchsize": false, // If use dynamic batch size
+        "ddp": false,
+        "train_stage": 1, // 0: train all modules, For VALL_E, support 1: AR Decoder 2: NAR Decoder(s)
+        "max_epoch": 20, 
+        "optimizer": "AdamW", 
+        "scheduler": "cosine",
+        "warmup_steps": 16000, // number of steps that affects how rapidly the learning rate decreases
+        "total_training_steps": 800000,
+        "base_lr": 1e-4, // base learning rate."
+        "valid_interval": 1000,
+        "log_epoch_step": 1000,
+        "save_checkpoint_stride": [
+            1,
+            1
+        ]
+    }
+}

config/vits.json

@@ -0,0 +1,101 @@
+{
+    "base_config": "config/tts.json",
+    "model_type": "VITS",
+    "task_type": "tts",
+    "preprocess": {
+        "extract_phone": true,
+        "extract_mel": true,
+        "n_mel": 80,
+        "fmin": 0,
+        "fmax": null,
+        "extract_linear_spec": true,
+        "extract_audio": true,
+        "use_linear": true,
+        "use_mel": true,
+        "use_audio": true,
+        "use_text": false,
+        "use_phone": true,
+        "lexicon_path": "./text/lexicon/librispeech-lexicon.txt",
+        "n_fft": 1024,
+        "win_size": 1024,
+        "hop_size": 256,
+        "segment_size": 8192,
+        "text_cleaners": [
+            "english_cleaners"
+        ]
+    },
+    "model": {
+        "text_token_num": 512,
+        "inter_channels": 192,
+        "hidden_channels": 192,
+        "filter_channels": 768,
+        "n_heads": 2,
+        "n_layers": 6,
+        "kernel_size": 3,
+        "p_dropout": 0.1,
+        "resblock": "1",
+        "resblock_kernel_sizes": [
+            3,
+            7,
+            11
+        ],
+        "resblock_dilation_sizes": [
+            [
+                1,
+                3,
+                5
+            ],
+            [
+                1,
+                3,
+                5
+            ],
+            [
+                1,
+                3,
+                5
+            ]
+        ],
+        "upsample_rates": [
+            8,
+            8,
+            2,
+            2
+        ],
+        "upsample_initial_channel": 512,
+        "upsample_kernel_sizes": [
+            16,
+            16,
+            4,
+            4
+        ],
+        "n_layers_q": 3,
+        "use_spectral_norm": false,
+        "n_speakers": 0, // number of speakers, while be automatically set if n_speakers is 0 and multi_speaker_training is true
+        "gin_channels": 256,
+        "use_sdp": true
+    },
+    "train": {
+        "fp16_run": true,
+        "learning_rate": 2e-4,
+        "betas": [
+            0.8,
+            0.99
+        ],
+        "eps": 1e-9,
+        "batch_size": 16,
+        "lr_decay": 0.999875,
+        // "segment_size": 8192,
+        "init_lr_ratio": 1,
+        "warmup_epochs": 0,
+        "c_mel": 45,
+        "c_kl": 1.0,
+        "AdamW": {
+            "betas": [
+                0.8,
+                0.99
+            ],
+            "eps": 1e-9,
+        }
+    }
+}

config/vitssvc.json

@@ -0,0 +1,306 @@
+{
+    "base_config": "config/svc/base.json",
+    "model_type": "VITS",
+    "task_type": "svc",
+    "preprocess": {
+        // Config for features extraction
+        "extract_mel": true,
+        "extract_pitch": true,
+        "pitch_extractor": "parselmouth",
+        "extract_energy": true,
+        "extract_uv": true,
+        "extract_linear_spec": true,
+        "extract_audio": true,
+        "mel_min_max_norm": true,
+        // Config for features usage
+        "use_linear": true,
+        "use_mel": true,
+        "use_min_max_norm_mel": false,
+        "use_audio": true,
+        "use_frame_pitch": true,
+        "use_uv": true,
+        "use_spkid": true,
+        "use_contentvec": false,
+        "use_whisper": false,
+        "use_wenet": false,
+        "use_text": false,
+        "use_phone": false,
+        "fmin": 0,
+        "fmax": 12000,
+        "f0_min": 50,
+        "f0_max": 1100,
+        // f0_bin in sovits
+        "pitch_bin": 256,
+        // filter_length in sovits
+        "n_fft": 1024,
+        // hop_length in sovits
+        "hop_size": 256,
+        // win_length in sovits
+        "win_size": 1024,
+        "segment_size": 8192,
+        "n_mel": 100,
+        "sample_rate": 24000,
+        "mel_min_max_stats_dir": "mel_min_max_stats",
+        "whisper_dir": "whisper",
+        "contentvec_dir": "contentvec",
+        "wenet_dir": "wenet",
+        "mert_dir": "mert",
+        // Meta file
+        "train_file": "train.json",
+        "valid_file": "test.json",
+        "spk2id": "singers.json",
+        "utt2spk": "utt2singer"
+    },
+    "model": {
+        "condition_encoder": {
+            "merge_mode": "add",
+            "input_melody_dim": 1,
+            "use_log_f0": true,
+            "n_bins_melody": 256,
+            "output_melody_dim": 384,
+            "input_loudness_dim": 1,
+            "use_log_loudness": true,
+            "n_bins_loudness": 256,
+            "output_loudness_dim": 384,
+            "use_whisper": false,
+            "use_contentvec": false,
+            "use_wenet": false,
+            "use_mert": false,
+            "whisper_dim": 1024,
+            "contentvec_dim": 256,
+            "mert_dim": 256,
+            "wenet_dim": 512,
+            "content_encoder_dim": 384,
+            "singer_table_size": 512,
+            "output_singer_dim": 384,
+            "output_content_dim": 384,
+            "use_spkid": true,
+            "pitch_max": 1100.0,
+            "pitch_min": 50.0,
+        },
+        "vits": {
+            "filter_channels": 256,
+            "gin_channels": 256,
+            "hidden_channels": 384,
+            "inter_channels": 384,
+            "kernel_size": 3,
+            "n_flow_layer": 4,
+            "n_heads": 2,
+            "n_layers": 6,
+            "n_layers_q": 3,
+            "n_speakers": 512,
+            "p_dropout": 0.1,
+            "use_spectral_norm": false,
+        },
+        "generator": "hifigan",
+        "generator_config": {
+            "hifigan": {
+                "resblock": "1",
+                "resblock_kernel_sizes": [
+                    3,
+                    7,
+                    11
+                ],
+                "upsample_rates": [
+                    8,
+                    8,
+                    2,
+                    2
+                ],
+                "upsample_kernel_sizes": [
+                    16,
+                    16,
+                    4,
+                    4
+                ],
+                "upsample_initial_channel": 512,
+                "resblock_dilation_sizes": [
+                    [
+                        1,
+                        3,
+                        5
+                    ],
+                    [
+                        1,
+                        3,
+                        5
+                    ],
+                    [
+                        1,
+                        3,
+                        5
+                    ]
+                ]
+            },
+            "melgan": {
+                "ratios": [
+                    8,
+                    8,
+                    2,
+                    2
+                ],
+                "ngf": 32,
+                "n_residual_layers": 3,
+                "num_D": 3,
+                "ndf": 16,
+                "n_layers": 4,
+                "downsampling_factor": 4
+            },
+            "bigvgan": {
+                "resblock": "1",
+                "activation": "snakebeta",
+                "snake_logscale": true,
+                "upsample_rates": [
+                    8,
+                    8,
+                    2,
+                    2
+                ],
+                "upsample_kernel_sizes": [
+                    16,
+                    16,
+                    4,
+                    4
+                ],
+                "upsample_initial_channel": 512,
+                "resblock_kernel_sizes": [
+                    3,
+                    7,
+                    11
+                ],
+                "resblock_dilation_sizes": [
+                    [
+                        1,
+                        3,
+                        5
+                    ],
+                    [
+                        1,
+                        3,
+                        5
+                    ],
+                    [
+                        1,
+                        3,
+                        5
+                    ]
+                ]
+            },
+            "nsfhifigan": {
+                "resblock": "1",
+                "harmonic_num": 8,
+                "upsample_rates": [
+                    8,
+                    8,
+                    2,
+                    2
+                ],
+                "upsample_kernel_sizes": [
+                    16,
+                    16,
+                    4,
+                    4
+                ],
+                "upsample_initial_channel": 768,
+                "resblock_kernel_sizes": [
+                    3,
+                    7,
+                    11
+                ],
+                "resblock_dilation_sizes": [
+                    [
+                        1,
+                        3,
+                        5
+                    ],
+                    [
+                        1,
+                        3,
+                        5
+                    ],
+                    [
+                        1,
+                        3,
+                        5
+                    ]
+                ]
+            },
+            "apnet": {
+                "ASP_channel": 512,
+                "ASP_resblock_kernel_sizes": [
+                    3,
+                    7,
+                    11
+                ],
+                "ASP_resblock_dilation_sizes": [
+                    [
+                        1,
+                        3,
+                        5
+                    ],
+                    [
+                        1,
+                        3,
+                        5
+                    ],
+                    [
+                        1,
+                        3,
+                        5
+                    ]
+                ],
+                "ASP_input_conv_kernel_size": 7,
+                "ASP_output_conv_kernel_size": 7,
+                "PSP_channel": 512,
+                "PSP_resblock_kernel_sizes": [
+                    3,
+                    7,
+                    11
+                ],
+                "PSP_resblock_dilation_sizes": [
+                    [
+                        1,
+                        3,
+                        5
+                    ],
+                    [
+                        1,
+                        3,
+                        5
+                    ],
+                    [
+                        1,
+                        3,
+                        5
+                    ]
+                ],
+                "PSP_input_conv_kernel_size": 7,
+                "PSP_output_R_conv_kernel_size": 7,
+                "PSP_output_I_conv_kernel_size": 7,
+            }
+        },
+    },
+    "train": {
+        "fp16_run": true,
+        "learning_rate": 2e-4,
+        "betas": [
+            0.8,
+            0.99
+        ],
+        "eps": 1e-9,
+        "batch_size": 16,
+        "lr_decay": 0.999875,
+        // "segment_size": 8192,
+        "init_lr_ratio": 1,
+        "warmup_epochs": 0,
+        "c_mel": 45,
+        "c_kl": 1.0,
+        "AdamW": {
+            "betas": [
+                0.8,
+                0.99
+            ],
+            "eps": 1e-9,
+        }
+    }
+}

config/vocoder.json

@@ -0,0 +1,84 @@
+{
+  "base_config": "config/base.json",
+  "dataset": [
+    "LJSpeech",
+    "LibriTTS",
+    "opencpop",
+    "m4singer",
+    "svcc",
+    "svcceval",
+    "pjs",
+    "opensinger",
+    "popbutfy",
+    "nus48e",
+    "popcs",
+    "kising",
+    "csd",
+    "opera",
+    "vctk",
+    "lijian",
+    "cdmusiceval"
+  ],
+  "task_type": "vocoder",
+  "preprocess": {
+    // acoustic features
+    "extract_mel": true,
+    "extract_pitch": false,
+    "extract_uv": false,
+    "extract_audio": true,
+    "extract_label": false,
+    "extract_one_hot": false,
+    "extract_amplitude_phase": false,
+    "pitch_extractor": "parselmouth",
+    // Settings for data preprocessing
+    "n_mel": 100,
+    "win_size": 1024,
+    "hop_size": 256,
+    "sample_rate": 24000,
+    "n_fft": 1024,
+    "fmin": 0,
+    "fmax": 12000,
+    "f0_min": 50,
+    "f0_max": 1100,
+    "pitch_bin": 256,
+    "pitch_max": 1100.0,
+    "pitch_min": 50.0,
+    "is_mu_law": false,
+    "bits": 8,
+    "cut_mel_frame": 32,
+    // Directory names of processed data or extracted features
+    "spk2id": "singers.json",
+    // Features used for model training
+    "use_mel": true,
+    "use_frame_pitch": false,
+    "use_uv": false,
+    "use_audio": true,
+    "use_label": false,
+    "use_one_hot": false,
+    "train_file": "train.json",
+    "valid_file": "test.json"
+  },
+  "train": {
+    "random_seed": 114514,
+    "batch_size": 64,
+    "gradient_accumulation_step": 1,
+    "max_epoch": 1000000,
+    "save_checkpoint_stride": [
+      20
+    ],
+    "run_eval": [
+      true
+    ],
+    "sampler": {
+      "holistic_shuffle": true,
+      "drop_last": true
+    },
+    "dataloader": {
+      "num_worker": 16,
+      "pin_memory": true
+    },
+    "tracker": [
+      "tensorboard"
+    ],
+  }
+}

egs/datasets/README.md

@@ -0,0 +1,458 @@
+# Datasets Format
+
+Amphion support the following academic datasets (sort alphabetically):
+
+- [Datasets Format](#datasets-format)
+  - [AudioCaps](#audiocaps)
+  - [CSD](#csd)
+  - [CustomSVCDataset](#customsvcdataset)
+  - [Hi-Fi TTS](#hifitts)
+  - [KiSing](#kising)
+  - [LibriLight](#librilight)
+  - [LibriTTS](#libritts)
+  - [LJSpeech](#ljspeech)
+  - [M4Singer](#m4singer)
+  - [NUS-48E](#nus-48e)
+  - [Opencpop](#opencpop)
+  - [OpenSinger](#opensinger)
+  - [Opera](#opera)
+  - [PopBuTFy](#popbutfy)
+  - [PopCS](#popcs)
+  - [PJS](#pjs)
+  - [SVCC](#svcc)
+  - [VCTK](#vctk)
+
+The downloading link and the file structure tree of each dataset is displayed as follows.
+
+> **Note:** When using Docker to run Amphion, mount the dataset to the container is necessary after downloading. Check [Mount dataset in Docker container](./docker.md) for more details.
+
+## AudioCaps
+
+AudioCaps is a dataset of around 44K audio-caption pairs, where each audio clip corresponds to a caption with rich semantic information.
+
+Download AudioCaps dataset [here](https://github.com/cdjkim/audiocaps). The file structure looks like below:
+
+```plaintext
+[AudioCaps dataset path]
+┣ AudioCpas
+┃ ┣ wav
+┃ ┃ ┣ ---1_cCGK4M_0_10000.wav
+┃ ┃ ┣ ---lTs1dxhU_30000_40000.wav
+┃ ┃ ┣ ...
+```
+
+## CSD
+
+Download the official CSD dataset [here](https://zenodo.org/records/4785016). The file structure looks like below:
+
+```plaintext
+[CSD dataset path]
+ ┣ english
+ ┣ korean
+ ┣ utterances
+ ┃ ┣ en001a
+ ┃ ┃ ┣ {UtterenceID}.wav
+ ┃ ┣ en001b
+ ┃ ┣ en002a
+ ┃ ┣ en002b
+ ┃ ┣ ...
+ ┣ README
+```
+
+## CustomSVCDataset
+
+We support custom dataset for Singing Voice Conversion. Organize your data in the following structure to construct your own dataset:
+
+```plaintext
+[Your Custom Dataset Path]
+ ┣ singer1
+ ┃ ┣ song1
+ ┃ ┃ ┣ utterance1.wav
+ ┃ ┃ ┣ utterance2.wav
+ ┃ ┃ ┣ ...
+ ┃ ┣ song2
+ ┃ ┣ ...
+ ┣ singer2
+ ┣ ...
+```
+
+
+## Hi-Fi TTS
+
+Download the official Hi-Fi TTS dataset [here](https://www.openslr.org/109/). The file structure looks like below:
+
+```plaintext
+[Hi-Fi TTS dataset path]
+ ┣ audio
+ ┃ ┣ 11614_other {Speaker_ID}_{SNR_subset}
+ ┃ ┃ ┣ 10547 {Book_ID}
+ ┃ ┃ ┃ ┣ thousandnights8_04_anonymous_0001.flac
+ ┃ ┃ ┃ ┣ thousandnights8_04_anonymous_0003.flac
+ ┃ ┃ ┃ ┣ thousandnights8_04_anonymous_0004.flac
+ ┃ ┃ ┃ ┣ ...
+ ┃ ┃ ┣ ...
+ ┃ ┣ ...
+ ┣ 92_manifest_clean_dev.json
+ ┣ 92_manifest_clean_test.json
+ ┣ 92_manifest_clean_train.json
+ ┣ ...
+ ┣ {Speaker_ID}_manifest_{SNR_subset}_{dataset_split}.json
+ ┣ ...
+ ┣ books_bandwidth.tsv
+ ┣ LICENSE.txt
+ ┣ readers_books_clean.txt
+ ┣ readers_books_other.txt
+ ┣ README.txt
+
+```
+
+## KiSing
+
+Download the official KiSing dataset [here](http://shijt.site/index.php/2021/05/16/kising-the-first-open-source-mandarin-singing-voice-synthesis-corpus/). The file structure looks like below:
+
+```plaintext
+[KiSing dataset path]
+ ┣ clean
+ ┃ ┣ 421
+ ┃ ┣ 422
+ ┃ ┣ ...
+```
+
+## LibriLight
+
+Download the official LibriLight dataset [here](https://github.com/facebookresearch/libri-light). The file structure looks like below:
+
+```plaintext
+[LibriTTS dataset path]
+ ┣ small (Subset)
+ ┃ ┣ 100 {Speaker_ID}
+ ┃ ┃ ┣ sea_fairies_0812_librivox_64kb_mp3 {Chapter_ID}
+ ┃ ┃ ┃ ┣ 01_baum_sea_fairies_64kb.flac
+ ┃ ┃ ┃ ┣ 02_baum_sea_fairies_64kb.flac
+ ┃ ┃ ┃ ┣ 03_baum_sea_fairies_64kb.flac
+ ┃ ┃ ┃ ┣ 22_baum_sea_fairies_64kb.flac
+ ┃ ┃ ┃ ┣ 01_baum_sea_fairies_64kb.json
+ ┃ ┃ ┃ ┣ 02_baum_sea_fairies_64kb.json
+ ┃ ┃ ┃ ┣ 03_baum_sea_fairies_64kb.json
+ ┃ ┃ ┃ ┣ 22_baum_sea_fairies_64kb.json
+ ┃ ┃ ┃ ┣ ...
+ ┃ ┃ ┣ ...
+ ┃ ┣ ...
+ ┣ medium (Subset)
+ ┣ ...
+```
+
+## LibriTTS
+
+Download the official LibriTTS dataset [here](https://www.openslr.org/60/). The file structure looks like below:
+
+```plaintext
+[LibriTTS dataset path]
+ ┣ BOOKS.txt
+ ┣ CHAPTERS.txt
+ ┣ eval_sentences10.tsv
+ ┣ LICENSE.txt
+ ┣ NOTE.txt
+ ┣ reader_book.tsv
+ ┣ README_librispeech.txt
+ ┣ README_libritts.txt 
+ ┣ speakers.tsv
+ ┣ SPEAKERS.txt
+ ┣ dev-clean (Subset)
+ ┃ ┣ 1272{Speaker_ID}
+ ┃ ┃ ┣ 128104 {Chapter_ID}
+ ┃ ┃ ┃ ┣ 1272_128104_000001_000000.normalized.txt
+ ┃ ┃ ┃ ┣ 1272_128104_000001_000000.original.txt
+ ┃ ┃ ┃ ┣ 1272_128104_000001_000000.wav
+ ┃ ┃ ┃ ┣ ...
+ ┃ ┃ ┃ ┣ 1272_128104.book.tsv
+ ┃ ┃ ┃ ┣ 1272_128104.trans.tsv
+ ┃ ┃ ┣ ...
+ ┃ ┣ ...
+ ┣ dev-other (Subset)
+ ┃ ┣ 116 (Speaker)
+ ┃ ┃ ┣ 288045 {Chapter_ID}
+ ┃ ┃ ┃ ┣ 116_288045_000003_000000.normalized.txt
+ ┃ ┃ ┃ ┣ 116_288045_000003_000000.original.txt
+ ┃ ┃ ┃ ┣ 116_288045_000003_000000.wav
+ ┃ ┃ ┃ ┣ ...
+ ┃ ┃ ┃ ┣ 116_288045.book.tsv
+ ┃ ┃ ┃ ┣ 116_288045.trans.tsv
+ ┃ ┃ ┣ ...
+ ┃ ┣ ...
+ ┃ ┣ ...
+ ┣ test-clean  (Subset)
+ ┃ �� {Speaker_ID}
+ ┃ ┃ ┣ {Chapter_ID}
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.normalized.txt
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.original.txt
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.wav
+ ┃ ┃ ┃ ┣ ...
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}.book.tsv
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}.trans.tsv
+ ┃ ┃ ┣ ...
+ ┃ ┣ ...
+ ┣ test-other
+ ┃ ┣ {Speaker_ID}
+ ┃ ┃ ┣ {Chapter_ID}
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.normalized.txt
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.original.txt
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.wav
+ ┃ ┃ ┃ ┣ ...
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}.book.tsv
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}.trans.tsv
+ ┃ ┃ ┣ ...
+ ┃ ┣ ...
+ ┣ train-clean-100
+ ┃ ┣ {Speaker_ID}
+ ┃ ┃ ┣ {Chapter_ID}
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.normalized.txt
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.original.txt
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.wav
+ ┃ ┃ ┃ ┣ ...
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}.book.tsv
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}.trans.tsv
+ ┃ ┃ ┣ ...
+ ┃ ┣ ...
+ ┣ train-clean-360
+ ┃ ┣ {Speaker_ID}
+ ┃ ┃ ┣ {Chapter_ID}
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.normalized.txt
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.original.txt
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.wav
+ ┃ ┃ ┃ ┣ ...
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}.book.tsv
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}.trans.tsv
+ ┃ ┃ ┣ ...
+ ┃ ┣ ...
+ ┣ train-other-500
+ ┃ ┣ {Speaker_ID}
+ ┃ ┃ ┣ {Chapter_ID}
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.normalized.txt
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.original.txt
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}_{Utterance_ID}.wav
+ ┃ ┃ ┃ ┣ ...
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}.book.tsv
+ ┃ ┃ ┃ ┣ {Speaker_ID}_{Chapter_ID}.trans.tsv
+ ┃ ┃ ┣ ...
+ ┃ ┣ ...
+```
+
+## LJSpeech
+
+Download the official LJSpeech dataset [here](https://keithito.com/LJ-Speech-Dataset/). The file structure looks like below:
+
+```plaintext
+[LJSpeech dataset path]
+ ┣ metadata.csv
+ ┣ wavs
+ ┃ ┣ LJ001-0001.wav
+ ┃ ┣ LJ001-0002.wav 
+ ┃ ┣ ...
+ ┣ README
+```
+
+## M4Singer
+
+Download the official M4Singer dataset [here](https://drive.google.com/file/d/1xC37E59EWRRFFLdG3aJkVqwtLDgtFNqW/view). The file structure looks like below:
+
+```plaintext
+[M4Singer dataset path]
+ ┣ {Singer_1}#{Song_1}
+ ┃ ┣ 0000.mid
+ ┃ ┣ 0000.TextGrid
+ ┃ ┣ 0000.wav
+ ┃ ┣ ...
+ ┣ {Singer_1}#{Song_2}
+ ┣ ...
+ ┣ {Singer_2}#{Song_1}
+ ┣ {Singer_2}#{Song_2}
+ ┣ ...
+ ┗ meta.json
+```
+
+## NUS-48E
+
+Download the official NUS-48E dataset [here](https://drive.google.com/drive/folders/12pP9uUl0HTVANU3IPLnumTJiRjPtVUMx). The file structure looks like below:
+
+```plaintext
+[NUS-48E dataset path]
+ ┣ {SpeakerID}
+ ┃ ┣ read
+ ┃ ┃ ┣ {SongID}.txt
+ ┃ ┃ ┣ {SongID}.wav
+ ┃ ┃ ┣ ...
+ ┃ ┣ sing
+ ┃ ┃ ┣ {SongID}.txt
+ ┃ ┃ ┣ {SongID}.wav
+ ┃ ┃ ┣ ...
+ ┣ ...
+ ┣ README.txt
+
+```
+
+## Opencpop
+
+Download the official Opencpop dataset [here](https://wenet.org.cn/opencpop/). The file structure looks like below:
+
+```plaintext
+[Opencpop dataset path]
+ ┣ midis
+ ┃ ┣ 2001.midi
+ ┃ ┣ 2002.midi
+ ┃ ┣ 2003.midi
+ ┃ ┣ ...
+ ┣ segments
+ ┃ ┣ wavs
+ ┃ ┃ ┣ 2001000001.wav
+ ┃ ┃ ┣ 2001000002.wav
+ ┃ ┃ ┣ 2001000003.wav
+ ┃ ┃ ┣ ...
+ ┃ ┣ test.txt
+ ┃ ┣ train.txt
+ ┃ ┗ transcriptions.txt
+ ┣ textgrids
+ ┃ ┣ 2001.TextGrid
+ ┃ ┣ 2002.TextGrid
+ ┃ ┣ 2003.TextGrid
+ ┃ ┣ ...
+ ┣ wavs
+ ┃ ┣ 2001.wav
+ ┃ ┣ 2002.wav
+ ┃ ┣ 2003.wav
+ ┃ ┣ ...
+ ┣ TERMS_OF_ACCESS
+ ┗ readme.md
+```
+
+## OpenSinger
+
+Download the official OpenSinger dataset [here](https://drive.google.com/file/d/1EofoZxvalgMjZqzUEuEdleHIZ6SHtNuK/view). The file structure looks like below:
+
+```plaintext
+[OpenSinger dataset path]
+ ┣ ManRaw
+ ┃ ┣ {Singer_1}_{Song_1}
+ ┃ ┃ ┣ {Singer_1}_{Song_1}_0.lab
+ ┃ ┃ ┣ {Singer_1}_{Song_1}_0.txt
+ ┃ ┃ ┣ {Singer_1}_{Song_1}_0.wav
+ ┃ ┃ ┣ ...
+ ┃ ┣ {Singer_1}_{Song_2}
+ ┃ ┣ ...
+ ┣ WomanRaw
+ ┣ LICENSE
+ ┗ README.md
+```
+
+## Opera
+
+Download the official Opera dataset [here](http://isophonics.net/SingingVoiceDataset). The file structure looks like below:
+
+```plaintext
+[Opera dataset path]
+ ┣ monophonic
+ ┃ ┣ chinese
+ ┃ ┃ ┣ {Gender}_{SingerID}
+ ┃ ┃ ┃ ┣ {Emotion}_{SongID}.wav
+ ┃ ┃ ┃ ┣ ...
+ ┃ ┃ ┣ ...
+ ┃ ┣ western
+ ┣ polyphonic
+ ┃ ┣ chinese
+ ┃ ┣ western
+ ┣ CrossculturalDataSet.xlsx
+```
+
+## PopBuTFy
+
+Download the official PopBuTFy dataset [here](https://github.com/MoonInTheRiver/NeuralSVB). The file structure looks like below:
+
+```plaintext
+[PopBuTFy dataset path]
+ ┣ data
+ ┃ ┣ {SingerID}#singing#{SongName}_Amateur
+ ┃ ┃ ┣ {SingerID}#singing#{SongName}_Amateur_{UtteranceID}.mp3
+ ┃ ┃ ┣ ...
+ ┃ ┣ {SingerID}#singing#{SongName}_Professional
+ ┃ ┃ ┣ {SingerID}#singing#{SongName}_Professional_{UtteranceID}.mp3
+ ┃ ┃ ┣ ...
+ ┣ text_labels
+ ┗ TERMS_OF_ACCESS
+```
+
+## PopCS
+
+Download the official PopCS dataset [here](https://github.com/MoonInTheRiver/DiffSinger/blob/master/resources/apply_form.md). The file structure looks like below:
+
+```plaintext
+[PopCS dataset path]
+ ┣ popcs
+ ┃ ┣ popcs-{SongName}
+ ┃ ┃ ┣ {UtteranceID}_ph.txt
+ ┃ ┃ ┣ {UtteranceID}_wf0.wav
+ ┃ ┃ ┣ {UtteranceID}.TextGrid
+ ┃ ┃ ┣ {UtteranceID}.txt
+ ┃ ┃ ┣ ...
+ ┃ ┣ ...
+ ┗ TERMS_OF_ACCESS
+```
+
+## PJS
+
+Download the official PJS dataset [here](https://sites.google.com/site/shinnosuketakamichi/research-topics/pjs_corpus). The file structure looks like below:
+
+```plaintext
+[PJS dataset path]
+ ┣ PJS_corpus_ver1.1
+ ┃ ┣ background_noise
+ ┃ ┣ pjs{SongID}
+ ┃ ┃ ┣ pjs{SongID}_song.wav
+ ┃ ┃ ┣ pjs{SongID}_speech.wav
+ ┃ ┃ ┣ pjs{SongID}.lab
+ ┃ ┃ ┣ pjs{SongID}.mid
+ ┃ ┃ ┣ pjs{SongID}.musicxml
+ ┃ ┃ ┣ pjs{SongID}.txt
+ ┃ ┣ ...
+```
+
+## SVCC
+
+Download the official SVCC dataset [here](https://github.com/lesterphillip/SVCC23_FastSVC/tree/main/egs/generate_dataset). The file structure looks like below:
+
+```plaintext
+[SVCC dataset path]
+ ┣ Data
+ ┃ ┣ CDF1
+ ┃ ┃ ┣ 10001.wav
+ ┃ ┃ ┣ 10002.wav
+ ┃ ┃ ┣ ...
+ ┃ ┣ CDM1
+ ┃ ┣ IDF1
+ ┃ ┣ IDM1
+ ┗ README.md
+```
+
+## VCTK
+
+Download the official VCTK dataset [here](https://datashare.ed.ac.uk/handle/10283/3443). The file structure looks like below:
+
+```plaintext
+[VCTK dataset path]
+ ┣ txt
+ ┃ ┣ {Speaker_1}
+ ┃ ┃ ┣ {Speaker_1}_001.txt
+ ┃ ┃ ┣ {Speaker_1}_002.txt
+ ┃ ┃ ┣ ...
+ ┃ ┣ {Speaker_2}
+ ┃ ┣ ...
+ ┣ wav48_silence_trimmed
+ ┃ ┣ {Speaker_1}
+ ┃ ┃ ┣ {Speaker_1}_001_mic1.flac
+ ┃ ┃ ┣ {Speaker_1}_001_mic2.flac
+ ┃ ┃ ┣ {Speaker_1}_002_mic1.flac
+ ┃ ┃ ┣ ...
+ ┃ ┣ {Speaker_2}
+ ┃ ┣ ...
+ ┣ speaker-info.txt
+ ┗ update.txt
+```

egs/datasets/docker.md

@@ -0,0 +1,19 @@
+# Mount dataset in Docker container
+
+When using Docker to run Amphion, mount the dataset to the container first is needed. It is recommend to mounte dataset to `/mnt/<dataset_name>` in the container, where `<dataset_name>` is the name of the dataset.
+
+When configuring the dataset in `exp_config.json`, you should use the path `/mnt/<dataset_name>` as the dataset path instead of the actual path on your host machine. Otherwise, the dataset will not be found in the container.
+
+## Mount Example
+
+```bash
+docker run --runtime=nvidia --gpus all -it -v .:/app -v <dataset_path1>:/mnt/<dataset_name1> -v <dataset_path2>:/mnt/<dataset_name2> amphion
+```
+
+For example, if you want to use the `LJSpeech` dataset, you can mount the dataset to `/mnt/LJSpeech` in the container.
+
+```bash
+docker run --runtime=nvidia --gpus all -it -v .:/app -v /home/username/datasets/LJSpeech:/mnt/LJSpeech amphion
+```
+
+If you want to use multiple datasets, you can mount them to different directories in the container by adding more `-v` options.

egs/metrics/README.md

@@ -0,0 +1,174 @@
+# Amphion Evaluation Recipe
+
+## Supported Evaluation Metrics
+
+Until now, Amphion Evaluation has supported the following objective metrics:
+
+- **F0 Modeling**:
+  - F0 Pearson Coefficients (FPC)
+  - F0 Periodicity Root Mean Square Error (PeriodicityRMSE)
+  - F0 Root Mean Square Error (F0RMSE)
+  - Voiced/Unvoiced F1 Score (V/UV F1)
+- **Energy Modeling**:
+  - Energy Root Mean Square Error (EnergyRMSE)
+  - Energy Pearson Coefficients (EnergyPC)
+- **Intelligibility**:
+  - Character Error Rate (CER) based on [Whipser](https://github.com/openai/whisper)
+  - Word Error Rate (WER) based on [Whipser](https://github.com/openai/whisper)
+- **Spectrogram Distortion**:
+  - Frechet Audio Distance (FAD)
+  - Mel Cepstral Distortion (MCD)
+  - Multi-Resolution STFT Distance (MSTFT)
+  - Perceptual Evaluation of Speech Quality (PESQ)
+  - Short Time Objective Intelligibility (STOI)
+  - Scale Invariant Signal to Distortion Ratio (SISDR)
+  - Scale Invariant Signal to Noise Ratio (SISNR)
+- **Speaker Similarity**:
+  - Cosine similarity based on:
+    - [Rawnet3](https://github.com/Jungjee/RawNet)
+    - [Resemblyzer](https://github.com/resemble-ai/Resemblyzer)
+    - [WavLM](https://huggingface.co/microsoft/wavlm-base-plus-sv)
+
+We provide a recipe to demonstrate how to objectively evaluate your generated audios. There are three steps in total:
+
+1. Pretrained Models Preparation
+2. Audio Data Preparation
+3. Evaluation
+
+## 1. Pretrained Models Preparation
+
+If you want to calculate `RawNet3` based speaker similarity, you need to download the pretrained model first, as illustrated [here](../../pretrained/README.md).
+
+## 2. Audio Data Preparation
+
+Prepare reference audios and generated audios in two folders, the `ref_dir` contains the reference audio and the `gen_dir` contains the generated audio. Here is an example.
+
+```plaintext
+ ┣ {ref_dir}
+ ┃ ┣ sample1.wav
+ ┃ ┣ sample2.wav
+ ┣ {gen_dir}
+ ┃ ┣ sample1.wav
+ ┃ ┣ sample2.wav
+```
+
+You have to make sure that the pairwise **reference audio and generated audio are named the same**, as illustrated above (sample1 to sample1, sample2 to sample2).
+
+## 3. Evaluation
+
+Run the `run.sh` with specified refenrece folder, generated folder, dump folder and metrics.
+
+```bash
+cd Amphion
+sh egs/metrics/run.sh \
+	--reference_folder [Your path to the reference audios] \
+	--generated_folder [Your path to the generated audios] \
+	--dump_folder [Your path to dump the objective results] \
+	--metrics [The metrics you need] \
+	--fs [Optional. To calculate all metrics in the specified sampling rate] \
+	--similarity_model [Optional. To choose the model for calculating the speaker similarity. Currently "rawnet", "wavlm" and "resemblyzer" are available. Default to "wavlm"] \
+	--similarity_mode [Optional. To choose the mode for calculating the speaker similarity. "pairwith" for calculating a series of ground truth / prediction audio pairs to obtain the speaker similarity, and "overall" for computing the average score with all possible pairs between the refernece folder and generated folder. Default to "pairwith"] \
+	--intelligibility_mode [Optionoal. To choose the mode for computing CER and WER. "gt_audio" means selecting the recognition content of the reference audio as the target, "gt_content" means using transcription as the target. Default to "gt_audio"] \
+	--ltr_path [Optional. Path to the transcription file] \
+	--language [Optional. Language for computing CER and WER. Default to "english"]
+```
+
+As for the metrics, an example is provided below:
+
+```bash
+--metrics "mcd pesq fad"
+```
+
+All currently available metrics keywords are listed below:
+
+| Keys                      | Description                                |
+| ------------------------- | ------------------------------------------ |
+| `fpc`                     | F0 Pearson Coefficients                    |
+| `f0_periodicity_rmse`     | F0 Periodicity Root Mean Square Error      |
+| `f0rmse`                  | F0 Root Mean Square Error                  |
+| `v_uv_f1`                 | Voiced/Unvoiced F1 Score                   |
+| `energy_rmse`             | Energy Root Mean Square Error              |
+| `energy_pc`               | Energy Pearson Coefficients                |
+| `cer`                     | Character Error Rate                       |
+| `wer`                     | Word Error Rate                            |
+| `similarity`      | Speaker Similarity
+| `fad`                     | Frechet Audio Distance                     |
+| `mcd`                     | Mel Cepstral Distortion                    |
+| `mstft`                   | Multi-Resolution STFT Distance             |
+| `pesq`                    | Perceptual Evaluation of Speech Quality    |
+| `si_sdr`                  | Scale Invariant Signal to Distortion Ratio |
+| `si_snr`                  | Scale Invariant Signal to Noise Ratio      |
+| `stoi`                    | Short Time Objective Intelligibility       |
+
+For example, if want to calculate the speaker similarity between the synthesized audio and the reference audio with the same content, run:
+
+```bash
+sh egs/metrics/run.sh \
+	--reference_folder [Your path to the reference audios] \
+	--generated_folder [Your path to the generated audios] \
+	--dump_folder [Your path to dump the objective results] \
+	--metrics "similarity" \
+	--similarity_model [Optional. To choose the model for calculating the speaker similarity. Currently "rawnet", "wavlm" and "resemblyzer" are available. Default to "wavlm"] \
+	--similarity_mode "pairwith" \
+```
+
+If you don't have the reference audio with the same content, run the following to get the conteng-free similarity score:
+
+```bash
+sh egs/metrics/run.sh \
+	--reference_folder [Your path to the reference audios] \
+	--generated_folder [Your path to the generated audios] \
+	--dump_folder [Your path to dump the objective results] \
+	--metrics "similarity" \
+	--similarity_model [Optional. To choose the model for calculating the speaker similarity. Currently "rawnet", "wavlm" and "resemblyzer" are available. Default to "wavlm"] \
+	--similarity_mode "overall" \
+```
+
+## Troubleshooting
+### FAD (Using Offline Models)
+If your system is unable to access huggingface.co from the terminal, you might run into an error like "OSError: Can't load tokenizer for ...". To work around this, follow these steps to use local models:
+
+1. Download the [bert-base-uncased](https://huggingface.co/bert-base-uncased), [roberta-base](https://huggingface.co/roberta-base), and [facebook/bart-base](https://huggingface.co/facebook/bart-base) models from `huggingface.co`. Ensure that the models are complete and uncorrupted. Place these directories within `Amphion/pretrained`. For a detailed file structure reference, see [This README](../../pretrained/README.md#optional-model-dependencies-for-evaluation) under `Amphion/pretrained`.
+2. Inside the `Amphion/pretrained` directory, create a bash script with the content outlined below. This script will automatically update the tokenizer paths used by your system:
+  ```bash
+  #!/bin/bash
+
+  BERT_DIR="bert-base-uncased"
+  ROBERTA_DIR="roberta-base"
+  BART_DIR="facebook/bart-base"
+  PYTHON_SCRIPT="[YOUR ENV PATH]/lib/python3.9/site-packages/laion_clap/training/data.py"
+
+  update_tokenizer_path() {
+      local dir_name=$1
+      local tokenizer_variable=$2
+      local full_path
+
+      if [ -d "$dir_name" ]; then
+          full_path=$(realpath "$dir_name")
+          if [ -f "$PYTHON_SCRIPT" ]; then
+              sed -i "s|${tokenizer_variable}.from_pretrained(\".*\")|${tokenizer_variable}.from_pretrained(\"$full_path\")|" "$PYTHON_SCRIPT"
+              echo "Updated ${tokenizer_variable} path to $full_path."
+          else
+              echo "Error: The specified Python script does not exist."
+              exit 1
+          fi
+      else
+          echo "Error: The directory $dir_name does not exist in the current directory."
+          exit 1
+      fi
+  }
+
+  update_tokenizer_path "$BERT_DIR" "BertTokenizer"
+  update_tokenizer_path "$ROBERTA_DIR" "RobertaTokenizer"
+  update_tokenizer_path "$BART_DIR" "BartTokenizer"
+
+  echo "BERT, BART and RoBERTa Python script paths have been updated."
+
+  ```
+
+3. The script provided is intended to adjust the tokenizer paths in the `data.py` file, found under `/lib/python3.9/site-packages/laion_clap/training/`, within your specific environment. For those utilizing conda, you can determine your environment path by running `conda info --envs`. Then, substitute `[YOUR ENV PATH]` in the script with this path. If your environment is configured differently, you'll need to update the `PYTHON_SCRIPT` variable to correctly point to the `data.py` file.
+4. Run the script. If it executes successfully, the tokenizer paths will be updated, allowing them to be loaded locally.
+
+### WavLM-based Speaker Similarity (Using Offline Models)
+
+If your system is unable to access huggingface.co from the terminal and you want to calculate `WavLM` based speaker similarity, you need to download the pretrained model first, as illustrated [here](../../pretrained/README.md).

egs/metrics/run.sh

@@ -0,0 +1,132 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+######## Build Experiment Environment ###########
+exp_dir=$(cd `dirname $0`; pwd)
+work_dir=$(dirname $(dirname $exp_dir))
+
+export WORK_DIR=$work_dir
+export PYTHONPATH=$work_dir
+export PYTHONIOENCODING=UTF-8
+
+######## Parse the Given Parameters from the Commond ###########
+options=$(getopt -o c:n:s --long gpu:,reference_folder:,generated_folder:,dump_folder:,metrics:,fs:,align_method:,energy_db_scale:,f0_subtract_mean:,similarity_model:,similarity_mode:,ltr_path:,intelligibility_mode:,language: -- "$@")
+eval set -- "$options"
+
+while true; do
+  case $1 in
+    # Visible GPU machines. The default value is "0".
+    --gpu) shift; gpu=$1 ; shift ;;
+    # Reference Audio Folder
+    --reference_folder) shift; ref_dir=$1 ; shift ;;
+    # Generated Audio Folder
+    --generated_folder) shift; deg_dir=$1 ; shift ;;
+    # Result Dumping Folder
+    --dump_folder) shift; dump_dir=$1 ; shift ;;
+    # Metrics to Compute
+    --metrics) shift; metrics=$1 ; shift ;;
+    # Sampling Rate
+    --fs) shift; fs=$1 ; shift ;;
+
+    # Method for aligning F0. The default value is "cut"
+    --align_method) shift; align_method=$1 ; shift ;;
+    # Method for normalizing F0. The default value is "True"
+    --f0_subtract_mean) shift; f0_subtract_mean=$1 ; shift ;;
+    # Method for normalizing Energy. The default value is "True"
+    --energy_db_scale) shift; energy_db_scale=$1 ; shift ;;
+
+    # Model for computing speaker similarity. The default value is "wavlm"
+    --similarity_model) shift; similarity_model=$1 ; shift ;;
+    # Mode for computing speaker similarity. The default value is "pairwith"
+    --similarity_mode) shift; similarity_mode=$1 ; shift ;;
+    
+    # Path for the transcript.
+    --ltr_path) shift; ltr_path=$1 ; shift ;;
+    # Mode for computing CER and WER. The default value is "gt_audio"
+    --intelligibility_mode) shift; intelligibility_mode=$1 ; shift ;;
+    # Language for computing CER and WER. The default value is "english"
+    --language) shift; language=$1 ; shift ;;
+
+    --) shift ; break ;;
+    *) echo "Invalid option: $1" exit 1 ;;
+  esac
+done
+
+### Value check ###
+if [ -z "$ref_dir" ]; then
+    echo "[Error] Please specify the reference_folder"
+    exit 1
+fi
+
+if [ -z "$deg_dir" ]; then
+    echo "[Error] Please specify the generated_folder"
+    exit 1
+fi
+
+if [ -z "$dump_dir" ]; then
+    echo "[Error] Please specify the dump_folder"
+    exit 1
+fi
+
+if [ -z "$metrics" ]; then
+    echo "[Error] Please specify the metrics"
+    exit 1
+fi
+
+if [ -z "$gpu" ]; then
+    gpu="0"
+fi
+
+if [ -z "$fs" ]; then
+    fs="None"
+fi
+
+if [ -z "$align_method" ]; then
+    align_method="dtw"
+fi
+
+if [ -z "$energy_db_scale" ]; then
+    energy_db_scale="True"
+fi
+
+if [ -z "$f0_subtract_mean" ]; then
+    f0_subtract_mean="True"
+fi
+
+if [ -z "$similarity_model" ]; then
+    similarity_model="wavlm"
+fi
+
+if [ -z "$similarity_mode" ]; then
+    similarity_mode="pairwith"
+fi
+
+if [ -z "$ltr_path" ]; then
+    ltr_path="None"
+fi
+
+if [ -z "$intelligibility_mode" ]; then
+    intelligibility_mode="gt_audio"
+fi
+
+if [ -z "$language" ]; then
+    language="english"
+fi
+
+######## Calculate Objective Metrics ###########
+CUDA_VISIBLE_DEVICES=$gpu python "$work_dir"/bins/calc_metrics.py \
+    --ref_dir $ref_dir \
+    --deg_dir $deg_dir \
+    --dump_dir $dump_dir \
+    --metrics $metrics \
+    --fs $fs \
+    --align_method $align_method \
+    --db_scale $energy_db_scale \
+    --f0_subtract_mean $f0_subtract_mean \
+    --similarity_model $similarity_model \
+    --similarity_mode $similarity_mode \
+    --ltr_path $ltr_path \
+    --intelligibility_mode $intelligibility_mode \
+    --language $language

egs/svc/DiffComoSVC/README.md

@@ -0,0 +1,234 @@
+# Accelerating Diffusion-based Singing Voice Conversion through Consistency Distillation
+<br>
+<div align="center">
+<img src="../../../imgs/svc/DiffComoSVC.png" width="90%">
+</div>
+<br>
+
+This is an implement of [Consistency Models](https://arxiv.org/abs/2303.01469) for accelerating diffusion-based singing voice conversion. The overall architecture follows "[Leveraging Content-based Features from Multiple Acoustic Models for Singing Voice Conversion](https://arxiv.org/abs/2310.11160)" (NeurIPS 2023 Workshop on Machine Learning for Audio), only a slightly modification is applied on acoustic model. Specifically,
+
+* The acoustic model is a conformer which generates a coarse spectrogram and a diffusion decoder based on Bidirectional Non-Causal Dilated CNN which polish the former spectrogram for better. This is similar to [CoMoSpeech: One-Step Speech and Singing Voice Synthesis via Consistency Model](https://comospeech.github.io/)
+* To accelerate diffusion model, we apply consistency distillation from [Consistency Models](https://arxiv.org/abs/2303.01469). For teacher model, the diffusion schedule of the diffusion decoder follows [karras diffusion](https://arxiv.org/abs/2206.00364). For distilling teacher model, the condition encoder and the conformer part of acoustic model are frozen while the diffusion decoder model is updated via exponential moving average. See Figure above for details.
+
+There are five stages in total:
+
+1. Data preparation
+2. Features extraction
+3. Teacher Model Training
+4. Consistency Distillation
+5. Inference/conversion
+
+## 1. Data Preparation
+
+### Dataset Download
+
+By default, we utilize the five datasets for training: M4Singer, Opencpop, OpenSinger, SVCC, and VCTK. How to download them is detailed [here](../../datasets/README.md).
+
+### Configuration
+
+Specify the dataset paths in  `exp_config.json`. Note that you can change the `dataset` list to use your preferred datasets.
+
+```json
+    "dataset": [
+        "m4singer",
+        "opencpop",
+        "opensinger",
+        "svcc",
+        "vctk"
+    ],
+    "dataset_path": {
+        // TODO: Fill in your dataset path
+        "m4singer": "[M4Singer dataset path]",
+        "opencpop": "[Opencpop dataset path]",
+        "opensinger": "[OpenSinger dataset path]",
+        "svcc": "[SVCC dataset path]",
+        "vctk": "[VCTK dataset path]"
+    },
+```
+
+## 2. Features Extraction
+
+### Content-based Pretrained Models Download
+
+By default, we utilize the Whisper and ContentVec to extract content features. How to download them is detailed [here](../../../pretrained/README.md).
+
+### Configuration
+
+Specify the dataset path and the output path for saving the processed data and the training model in `exp_config.json`:
+
+```json
+    // TODO: Fill in the output log path
+    "log_dir": "[Your path to save logs and checkpoints]",
+    "preprocess": {
+        // TODO: Fill in the output data path
+        "processed_dir": "[Your path to save processed data]",
+        ...
+    },
+```
+
+### Run
+
+Run the `run.sh` as the preproces stage (set  `--stage 1`).
+
+```bash
+cd Amphion
+sh egs/svc/DiffComoSVC/run.sh --stage 1
+```
+
+Note: The `CUDA_VISIBLE_DEVICES` is set as `"0"` in default. You can change it when running `run.sh` by specifying such as `--gpu "1"`.
+
+## 3. Teacher Model Training
+
+### Configuration
+
+Set the `distill` in `config/comosvc.json` to `false` for teacher model training, you can also specify the detailed configuration for conformer encoder and diffusion process here: 
+
+```JSON
+"comosvc":{
+            "distill": false,
+            // conformer encoder
+            "input_dim": 384,
+            "output_dim": 100,
+            "n_heads": 2,
+            "n_layers": 6,
+            "filter_channels":512,
+            // karras diffusion
+            "P_mean": -1.2,
+            "P_std": 1.2,
+            "sigma_data": 0.5,
+            "sigma_min": 0.002,
+            "sigma_max": 80,
+            "rho": 7,
+            "n_timesteps": 40,
+        },
+```
+
+We provide the default hyparameters in the `exp_config.json`. They can work on single NVIDIA-24g GPU. You can adjust them based on you GPU machines.
+
+```json
+"train": {
+        "batch_size": 32,
+        ...
+        "adamw": {
+            "lr": 2.0e-4
+        },
+        ...
+    }
+```
+
+### Run
+
+Run the `run.sh` as the training stage (set  `--stage 2`). Specify a experimental name to run the following command. The tensorboard logs and checkpoints will be saved in `[Your path to save logs and checkpoints]/[YourExptName]`.
+
+```bash
+cd Amphion
+sh egs/svc/DiffComoSVC/run.sh --stage 2 --name [YourExptName]
+```
+
+Note: The `CUDA_VISIBLE_DEVICES` is set as `"0"` in default. You can specify it when running `run.sh` such as:
+
+```bash
+cd Amphion
+sh egs/svc/DiffComoSVC/run.sh --stage 2 --name [YourExptName] --gpu "0,1,2,3"
+```
+
+## 4. Consistency Distillation
+
+### Configuration
+
+Set the `distill` in `config/comosvc.json` to `true` for teacher model training, and specify the `teacher_model_path` for consistency distillation. You can also specify the detailed configuration for conformer encoder and diffusion process here: 
+
+```JSON
+"model": {
+    "teacher_model_path":"[Your_teacher_model_checkpoint].bin",
+    ...
+    "comosvc":{
+                "distill": true,
+                // conformer encoder
+                "input_dim": 384,
+                "output_dim": 100,
+                "n_heads": 2,
+                "n_layers": 6,
+                "filter_channels":512,
+                // karras diffusion
+                "P_mean": -1.2,
+                "P_std": 1.2,
+                "sigma_data": 0.5,
+                "sigma_min": 0.002,
+                "sigma_max": 80,
+                "rho": 7,
+                "n_timesteps": 40,
+            },
+```
+
+We provide the default hyparameters in the `exp_config.json`. They can work on single NVIDIA-24g GPU. You can adjust them based on you GPU machines.
+
+```json
+"train": {
+        "batch_size": 32,
+        ...
+        "adamw": {
+            "lr": 2.0e-4
+        },
+        ...
+    }
+```
+
+### Run
+
+Run the `run.sh` as the training stage (set  `--stage 2`). Specify a experimental name to run the following command. The tensorboard logs and checkpoints will be saved in `[Your path to save logs and checkpoints]/[YourExptName]`.
+
+```bash
+cd Amphion
+sh egs/svc/DiffComoSVC/run.sh --stage 2 --name [YourExptName]
+```
+
+Note: The `CUDA_VISIBLE_DEVICES` is set as `"0"` in default. You can specify it when running `run.sh` such as:
+
+```bash
+cd Amphion
+sh egs/svc/DiffComoSVC/run.sh --stage 2 --name [YourExptName] --gpu "0,1,2,3"
+```
+
+## 5. Inference/Conversion
+
+### Pretrained Vocoder Download
+
+We fine-tune the official BigVGAN pretrained model with over 120 hours singing voice data. The benifits of fine-tuning has been investigated in our paper (see this [demo page](https://www.zhangxueyao.com/data/MultipleContentsSVC/vocoder.html)). The final pretrained singing voice vocoder is released [here](../../../pretrained/README.md#amphion-singing-bigvgan) (called `Amphion Singing BigVGAN`).
+
+### Run
+
+For inference/conversion, you need to specify the following configurations when running `run.sh`:
+
+| Parameters                                          | Description                                                  | Example                                                      |
+| --------------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| `--infer_expt_dir`                                  | The experimental directory which contains `checkpoint`       | `[Your path to save logs and checkpoints]/[YourExptName]`    |
+| `--infer_output_dir`                                | The output directory to save inferred audios.                | `[Your path to save logs and checkpoints]/[YourExptName]/result` |
+| `--infer_source_file` or `--infer_source_audio_dir` | The inference source (can be a json file or a dir).          | The `infer_source_file` could be `[Your path to save processed data]/[YourDataset]/test.json`, and the `infer_source_audio_dir` is a folder which includes several audio files (*.wav, *.mp3 or *.flac). |
+| `--infer_target_speaker`                            | The target speaker you want to convert into. You can refer to `[Your path to save logs and checkpoints]/[YourExptName]/singers.json` to choose a trained speaker. | For opencpop dataset, the speaker name would be `opencpop_female1`. |
+| `--infer_key_shift`                                 | How many semitones you want to transpose.                    | `"autoshfit"` (by default), `3`, `-3`, etc.                  |
+
+For example, if you want to make `opencpop_female1` sing the songs in the `[Your Audios Folder]`, just run:
+
+```bash
+cd Amphion
+sh egs/svc/DiffComoSVC/run.sh --stage 3 --gpu "0" \
+    --infer_expt_dir [Your path to save logs and checkpoints]/[YourExptName] \
+    --infer_output_dir [Your path to save logs and checkpoints]/[YourExptName]/result \
+    --infer_source_audio_dir [Your Audios Folder] \
+    --infer_target_speaker "opencpop_female1" \
+    --infer_key_shift "autoshift"
+```
+Specially, you can configurate the inference steps for teacher model by setting `inference` at `exp_config`(student model is always one-step sampling):
+```json
+    "inference": {
+        "comosvc": {
+            "inference_steps": 40
+        }
+    }
+```
+
+# Reference
+https://github.com/zhenye234/CoMoSpeech
+
+https://github.com/openai/consistency_models

egs/svc/DiffComoSVC/exp_config.json

@@ -0,0 +1,143 @@
+{
+    "base_config": "config/comosvc.json",
+    "model_type": "DiffComoSVC",
+    "dataset": [
+        "m4singer",
+        "opencpop",
+        "opensinger",
+        "svcc",
+        "vctk"
+    ],
+    "dataset_path": {
+        // TODO: Fill in your dataset path
+        "m4singer": "[M4Singer dataset path]",
+        "opencpop": "[Opencpop dataset path]",
+        "opensinger": "[OpenSinger dataset path]",
+        "svcc": "[SVCC dataset path]",
+        "vctk": "[VCTK dataset path]"
+    },
+    // TODO: Fill in the output log path
+    "log_dir": "[Your path to save logs and checkpoints]",
+    "preprocess": {
+        // TODO: Fill in the output data path
+        "processed_dir": "[Your path to save processed data]",
+        // Config for features extraction
+        "extract_mel": true,
+        "extract_pitch": true,
+        "extract_energy": true,
+        "extract_whisper_feature": true,
+        "extract_contentvec_feature": true,
+        "extract_wenet_feature": false,
+        "whisper_batch_size": 30, // decrease it if your GPU is out of memory
+        "contentvec_batch_size": 1,
+        // Fill in the content-based pretrained model's path
+        "contentvec_file": "pretrained/contentvec/checkpoint_best_legacy_500.pt",
+        "wenet_model_path": "pretrained/wenet/20220506_u2pp_conformer_exp/final.pt",
+        "wenet_config": "pretrained/wenet/20220506_u2pp_conformer_exp/train.yaml",
+        "whisper_model": "medium",
+        "whisper_model_path": "pretrained/whisper/medium.pt",
+        // Config for features usage
+        "use_mel": true,
+        "use_min_max_norm_mel": true,
+        "use_frame_pitch": true,
+        "use_frame_energy": true,
+        "use_spkid": true,
+        "use_whisper": true,
+        "use_contentvec": true,
+        "use_wenet": false,
+        "n_mel": 100,
+        "sample_rate": 24000
+    },
+    "model": {
+        "teacher_model_path":"[Your_teacher_model_checkpoint].bin",
+        "condition_encoder": {
+            // Config for features usage
+            "use_whisper": true,
+            "use_contentvec": true,
+            "use_wenet": false,
+            "whisper_dim": 1024,
+            "contentvec_dim": 256,
+            "wenet_dim": 512,
+            "use_singer_encoder": false,
+            "pitch_min": 50,
+            "pitch_max": 1100
+        },
+        "comosvc":{
+            "distill": false,
+            // conformer encoder
+            "input_dim": 384,
+            "output_dim": 100,
+            "n_heads": 2,
+            "n_layers": 6,
+            "filter_channels":512,
+            "dropout":0.1,
+            // karras diffusion
+            "P_mean": -1.2,
+            "P_std": 1.2,
+            "sigma_data": 0.5,
+            "sigma_min": 0.002,
+            "sigma_max": 80,
+            "rho": 7,
+            "n_timesteps": 40,
+        },
+        "diffusion": {
+            // Diffusion steps encoder
+            "step_encoder": {
+                "dim_raw_embedding": 128,
+                "dim_hidden_layer": 512,
+                "activation": "SiLU",
+                "num_layer": 2,
+                "max_period": 10000
+            },
+            // Diffusion decoder
+            "model_type": "bidilconv",
+            // bidilconv, unet2d, TODO: unet1d
+            "bidilconv": {
+                "base_channel": 384,
+                "n_res_block": 20,
+                "conv_kernel_size": 3,
+                "dilation_cycle_length": 4,
+                // specially, 1 means no dilation
+                "conditioner_size": 100
+            }
+        }
+    },
+    "train": {
+        "batch_size": 64,
+        "gradient_accumulation_step": 1,
+        "max_epoch": -1, // -1 means no limit
+        "save_checkpoint_stride": [
+            50,
+            50
+        ],
+        "keep_last": [
+            5,
+            -1
+        ],
+        "run_eval": [
+            false,
+            true
+        ],
+        "adamw": {
+            "lr": 4.0e-4
+        },
+        "reducelronplateau": {
+            "factor": 0.8,
+            "patience": 10,
+            "min_lr": 1.0e-4
+        },
+        "dataloader": {
+            "num_worker": 8,
+            "pin_memory": true
+        },
+        "sampler": {
+            "holistic_shuffle": false,
+            "drop_last": true
+        }
+    },
+    "inference": {
+        "comosvc": {
+            "inference_steps": 40
+        }
+    }
+}

egs/svc/DiffComoSVC/run.sh

@@ -0,0 +1 @@
+../_template/run.sh

egs/svc/MultipleContentsSVC/README.md

@@ -0,0 +1,248 @@
+# Leveraging Content-based Features from Multiple Acoustic Models for Singing Voice Conversion
+
+[![arXiv](https://img.shields.io/badge/arXiv-Paper-<COLOR>.svg)](https://arxiv.org/abs/2310.11160)
+[![demo](https://img.shields.io/badge/SVC-Demo-red)](https://www.zhangxueyao.com/data/MultipleContentsSVC/index.html)
+[![hf](https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace-Models-pink)](https://huggingface.co/amphion/singing_voice_conversion)
+[![hf](https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace-Spaces-yellow)](https://huggingface.co/spaces/amphion/singing_voice_conversion)
+[![openxlab](https://cdn-static.openxlab.org.cn/app-center/openxlab_app.svg)](https://openxlab.org.cn/apps/detail/Amphion/singing_voice_conversion)
+
+<br>
+<div align="center">
+<img src="../../../imgs/svc/MultipleContentsSVC.png" width="85%">
+</div>
+<br>
+
+This is the official implementation of the paper "[Leveraging Content-based Features from Multiple Acoustic Models for Singing Voice Conversion](https://arxiv.org/abs/2310.11160)" (NeurIPS 2023 Workshop on Machine Learning for Audio). Specially,
+
+- The muptile content features are from [Whipser](https://github.com/wenet-e2e/wenet) and [ContentVec](https://github.com/auspicious3000/contentvec).
+- The acoustic model is based on Bidirectional Non-Causal Dilated CNN (called `DiffWaveNetSVC` in Amphion), which is similar to [WaveNet](https://arxiv.org/pdf/1609.03499.pdf), [DiffWave](https://openreview.net/forum?id=a-xFK8Ymz5J), and [DiffSVC](https://ieeexplore.ieee.org/document/9688219).
+- The vocoder is [BigVGAN](https://github.com/NVIDIA/BigVGAN) architecture and we fine-tuned it in over 120 hours singing voice data.
+
+## A Little Taste Before Getting Started
+
+Before you delve into the code, we suggest exploring the interactive DEMO we've provided for a comprehensive overview. There are several ways you can engage with it:
+
+1. **Online DEMO**
+	
+	|                         HuggingFace                          |                           OpenXLab                           |
+	| :----------------------------------------------------------: | :----------------------------------------------------------: |
+	| [![hf](https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace-Spaces-yellow)](https://huggingface.co/spaces/amphion/singing_voice_conversion)<br />(Worldwide) | [![openxlab](https://cdn-static.openxlab.org.cn/app-center/openxlab_app.svg)](https://openxlab.org.cn/apps/detail/Amphion/singing_voice_conversion)<br />(Suitable for Mainland China Users) |
+
+2. **Run Local Gradio DEMO**
+
+	|                       Run with Docker                        |               Duplicate Space with Private GPU               |
+	| :----------------------------------------------------------: | :----------------------------------------------------------: |
+	| [![hf](https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace-Spaces-yellow)](https://huggingface.co/spaces/amphion/singing_voice_conversion?docker=true) | [![hf](https://img.shields.io/badge/%F0%9F%A4%97%20HuggingFace-Spaces-yellow)](https://huggingface.co/spaces/amphion/singing_voice_conversion?duplicate=true) |
+
+3. **Run with the Extended Colab**
+
+	You can check out [this repo](https://github.com/camenduru/singing-voice-conversion-colab) to run it with Colab. Thanks to [@camenduru](https://x.com/camenduru?s=20) and the community for their support!
+
+## Usage Overview
+
+To train a `DiffWaveNetSVC` model, there are four stages in total:
+
+1. Data preparation
+2. Features extraction
+3. Training
+4. Inference/conversion
+
+> **NOTE:** You need to run every command of this recipe in the `Amphion` root path:
+> ```bash
+> cd Amphion
+> ```
+
+## 1. Data Preparation
+
+### Dataset Download
+
+By default, we utilize the five datasets for training: M4Singer, Opencpop, OpenSinger, SVCC, and VCTK. How to download them is detailed [here](../../datasets/README.md).
+
+### Configuration
+
+Specify the dataset paths in  `exp_config.json`. Note that you can change the `dataset` list to use your preferred datasets.
+
+```json
+    "dataset": [
+        "m4singer",
+        "opencpop",
+        "opensinger",
+        "svcc",
+        "vctk"
+    ],
+    "dataset_path": {
+        // TODO: Fill in your dataset path
+        "m4singer": "[M4Singer dataset path]",
+        "opencpop": "[Opencpop dataset path]",
+        "opensinger": "[OpenSinger dataset path]",
+        "svcc": "[SVCC dataset path]",
+        "vctk": "[VCTK dataset path]"
+    },
+```
+
+### Custom Dataset
+
+We support custom dataset, see [here](../../datasets/README.md#customsvcdataset) for the file structure to follow.
+
+After constructing proper file structure, specify your dataset name in `dataset` and its path in `dataset_path`, also add its name in `use_custom_dataset`:
+
+```json
+    "dataset": [
+        "[Exisiting Dataset Name]",
+        //...
+        "[Your Custom Dataset Name]"
+    ],
+    "dataset_path": {
+        "[Exisiting Dataset Name]": "[Exisiting Dataset Path]",
+        //...
+        "[Your Custom Dataset Name]": "[Your Custom Dataset Path]"
+    },
+    "use_custom_dataset": [
+        "[Your Custom Dataset Name]"
+    ],
+```
+
+> **NOTE:** Custom dataset name does not have to be the same as the folder name. But it needs to satisfy these rules:
+> 1. It can not be the same as the exisiting dataset name.
+> 2. It can not contain any space or underline(`_`).
+> 3. It must be a valid folder name for operating system.
+> 
+> Some examples of valid custom dataset names are `mydataset`, `myDataset`, `my-dataset`, `mydataset1`, `my-dataset-1`, etc.
+
+## 2. Features Extraction
+
+### Content-based Pretrained Models Download
+
+By default, we utilize the Whisper and ContentVec to extract content features. How to download them is detailed [here](../../../pretrained/README.md).
+
+### Configuration
+
+Specify the dataset path and the output path for saving the processed data and the training model in `exp_config.json`:
+
+```json
+    // TODO: Fill in the output log path. The default value is "Amphion/ckpts/svc"
+    "log_dir": "ckpts/svc",
+    "preprocess": {
+        // TODO: Fill in the output data path. The default value is "Amphion/data"
+        "processed_dir": "data",
+        ...
+    },
+```
+
+### Run
+
+Run the `run.sh` as the preproces stage (set  `--stage 1`).
+
+```bash
+sh egs/svc/MultipleContentsSVC/run.sh --stage 1
+```
+
+> **NOTE:** The `CUDA_VISIBLE_DEVICES` is set as `"0"` in default. You can change it when running `run.sh` by specifying such as `--gpu "1"`.
+
+## 3. Training
+
+### Configuration
+
+We provide the default hyparameters in the `exp_config.json`. They can work on single NVIDIA-24g GPU. You can adjust them based on you GPU machines.
+
+```json
+"train": {
+        "batch_size": 32,
+        ...
+        "adamw": {
+            "lr": 2.0e-4
+        },
+        ...
+    }
+```
+
+### Train From Scratch
+
+Run the `run.sh` as the training stage (set  `--stage 2`). Specify a experimental name to run the following command. The tensorboard logs and checkpoints will be saved in `Amphion/ckpts/svc/[YourExptName]`.
+
+```bash
+sh egs/svc/MultipleContentsSVC/run.sh --stage 2 --name [YourExptName]
+```
+
+### Train From Existing Source
+
+We support training from existing source for various purposes. You can resume training the model from a checkpoint or fine-tune a model from another checkpoint.
+
+Setting `--resume true`, the training will resume from the **latest checkpoint** by default. For example, if you want to resume training from the latest checkpoint in `Amphion/ckpts/svc/[YourExptName]/checkpoint`, run:
+
+```bash
+sh egs/svc/MultipleContentsSVC/run.sh --stage 2 --name [YourExptName] \
+    --resume true
+```
+
+You can choose a **specific checkpoint** for retraining by `--resume_from_ckpt_path` argument. For example, if you want to fine-tune from the checkpoint `Amphion/ckpts/svc/[YourExptName]/checkpoint/[SpecificCheckpoint]`, run:
+
+```bash
+sh egs/svc/MultipleContentsSVC/run.sh --stage 2 --name [YourExptName] \
+    --resume true
+    --resume_from_ckpt_path "Amphion/ckpts/svc/[YourExptName]/checkpoint/[SpecificCheckpoint]" \
+```
+
+If you want to **fine-tune from another checkpoint**, just use `--resume_type` and set it to `"finetune"`. For example, If you want to fine-tune from the checkpoint `Amphion/ckpts/svc/[AnotherExperiment]/checkpoint/[SpecificCheckpoint]`, run:
+
+```bash
+sh egs/svc/MultipleContentsSVC/run.sh --stage 2 --name [YourExptName] \
+    --resume true
+    --resume_from_ckpt_path "Amphion/ckpts/svc/[AnotherExperiment]/checkpoint/[SpecificCheckpoint]" \
+    --resume_type "finetune"
+```
+
+> **NOTE:** The `--resume_type` is set as `"resume"` in default. It's not necessary to specify it when resuming training.
+> 
+> The difference between `"resume"` and `"finetune"` is that the `"finetune"` will **only** load the pretrained model weights from the checkpoint, while the `"resume"` will load all the training states (including optimizer, scheduler, etc.) from the checkpoint.
+
+Here are some example scenarios to better understand how to use these arguments:
+| Scenario | `--resume` | `--resume_from_ckpt_path` | `--resume_type` |
+| ------ | -------- | ----------------------- | ------------- |
+| You want to train from scratch | no | no | no |
+| The machine breaks down during training and you want to resume training from the latest checkpoint | `true` | no | no |
+| You find the latest model is overfitting and you want to re-train from the checkpoint before | `true` | `SpecificCheckpoint Path` | no |
+| You want to fine-tune a model from another checkpoint | `true` | `SpecificCheckpoint Path` | `"finetune"` |
+
+
+> **NOTE:** The `CUDA_VISIBLE_DEVICES` is set as `"0"` in default. You can change it when running `run.sh` by specifying such as `--gpu "0,1,2,3"`.
+
+## 4. Inference/Conversion
+
+### Pretrained Vocoder Download
+
+We fine-tune the official BigVGAN pretrained model with over 120 hours singing voice data. The benifits of fine-tuning has been investigated in our paper (see this [demo page](https://www.zhangxueyao.com/data/MultipleContentsSVC/vocoder.html)). The final pretrained singing voice vocoder is released [here](../../../pretrained/README.md#amphion-singing-bigvgan) (called `Amphion Singing BigVGAN`).
+
+### Run
+
+For inference/conversion, you need to specify the following configurations when running `run.sh`:
+
+| Parameters                                          | Description                                                                                                                                | Example                                                                                                                                                                            |
+| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `--infer_expt_dir`                                  | The experimental directory which contains `checkpoint`                                                                                     | `Amphion/ckpts/svc/[YourExptName]`                                                                                                                                                 |
+| `--infer_output_dir`                                | The output directory to save inferred audios.                                                                                              | `Amphion/ckpts/svc/[YourExptName]/result`                                                                                                                                          |
+| `--infer_source_file` or `--infer_source_audio_dir` | The inference source (can be a json file or a dir).                                                                                        | The `infer_source_file` could be `Amphion/data/[YourDataset]/test.json`, and the `infer_source_audio_dir` is a folder which includes several audio files (*.wav, *.mp3 or *.flac). |
+| `--infer_target_speaker`                            | The target speaker you want to convert into. You can refer to `Amphion/ckpts/svc/[YourExptName]/singers.json` to choose a trained speaker. | For opencpop dataset, the speaker name would be `opencpop_female1`.                                                                                                                |
+| `--infer_key_shift`                                 | How many semitones you want to transpose.                                                                                                  | `"autoshfit"` (by default), `3`, `-3`, etc.                                                                                                                                        |
+
+For example, if you want to make `opencpop_female1` sing the songs in the `[Your Audios Folder]`, just run:
+
+```bash
+sh egs/svc/MultipleContentsSVC/run.sh --stage 3 --gpu "0" \
+	--infer_expt_dir ckpts/svc/[YourExptName] \
+	--infer_output_dir ckpts/svc/[YourExptName]/result \
+	--infer_source_audio_dir [Your Audios Folder] \
+	--infer_target_speaker "opencpop_female1" \
+	--infer_key_shift "autoshift"
+```
+
+## Citations
+
+```bibtex
+@article{zhang2023leveraging,
+  title={Leveraging Content-based Features from Multiple Acoustic Models for Singing Voice Conversion},
+  author={Zhang, Xueyao and Gu, Yicheng and Chen, Haopeng and Fang, Zihao and Zou, Lexiao and Xue, Liumeng and Wu, Zhizheng},
+  journal={Machine Learning for Audio Workshop, NeurIPS 2023},
+  year={2023}
+}
+```

egs/svc/MultipleContentsSVC/exp_config.json

@@ -0,0 +1,127 @@
+{
+    "base_config": "config/svc/diffusion.json",
+    "model_type": "DiffWaveNetSVC",
+    "dataset": [
+        "m4singer",
+        "opencpop",
+        "opensinger",
+        "svcc",
+        "vctk"
+    ],
+    "dataset_path": {
+        // TODO: Fill in your dataset path
+        "m4singer": "[M4Singer dataset path]",
+        "opencpop": "[Opencpop dataset path]",
+        "opensinger": "[OpenSinger dataset path]",
+        "svcc": "[SVCC dataset path]",
+        "vctk": "[VCTK dataset path]"
+    },
+    // TODO: Fill in the output log path. The default value is "Amphion/ckpts/svc"
+    "log_dir": "ckpts/svc",
+    "preprocess": {
+        // TODO: Fill in the output data path. The default value is "Amphion/data"
+        "processed_dir": "data",
+        // Config for features extraction
+        "features_extraction_mode": "offline", // Online or offline features extraction ("offline" or "online")
+        "extract_mel": true,
+        "extract_pitch": true,
+        "extract_energy": true,
+        "extract_whisper_feature": true,
+        "extract_contentvec_feature": true,
+        "extract_wenet_feature": false,
+        "whisper_batch_size": 30, // decrease it if your GPU is out of memory
+        "contentvec_batch_size": 1,
+        // Fill in the content-based pretrained model's path
+        "contentvec_file": "pretrained/contentvec/checkpoint_best_legacy_500.pt",
+        "wenet_model_path": "pretrained/wenet/20220506_u2pp_conformer_exp/final.pt",
+        "wenet_config": "pretrained/wenet/20220506_u2pp_conformer_exp/train.yaml",
+        "whisper_model": "medium",
+        "whisper_model_path": "pretrained/whisper/medium.pt",
+        // Config for features usage
+        "use_mel": true,
+        "use_min_max_norm_mel": true,
+        "use_frame_pitch": true,
+        "use_frame_energy": true,
+        "use_spkid": true,
+        "use_whisper": true,
+        "use_contentvec": true,
+        "use_wenet": false,
+        "n_mel": 100,
+        "sample_rate": 24000
+    },
+    "model": {
+        "condition_encoder": {
+            // Config for features usage
+            "use_whisper": true,
+            "use_contentvec": true,
+            "use_wenet": false,
+            "whisper_dim": 1024,
+            "contentvec_dim": 256,
+            "wenet_dim": 512,
+            "use_singer_encoder": false,
+            "pitch_min": 50,
+            "pitch_max": 1100
+        },
+        "diffusion": {
+            "scheduler": "ddpm",
+            "scheduler_settings": {
+                "num_train_timesteps": 1000,
+                "beta_start": 1.0e-4,
+                "beta_end": 0.02,
+                "beta_schedule": "linear"
+            },
+            // Diffusion steps encoder
+            "step_encoder": {
+                "dim_raw_embedding": 128,
+                "dim_hidden_layer": 512,
+                "activation": "SiLU",
+                "num_layer": 2,
+                "max_period": 10000
+            },
+            // Diffusion decoder
+            "model_type": "bidilconv",
+            // bidilconv, unet2d, TODO: unet1d
+            "bidilconv": {
+                "base_channel": 512,
+                "n_res_block": 40,
+                "conv_kernel_size": 3,
+                "dilation_cycle_length": 4,
+                // specially, 1 means no dilation
+                "conditioner_size": 384
+            }
+        }
+    },
+    "train": {
+        "batch_size": 32,
+        "gradient_accumulation_step": 1,
+        "max_epoch": -1, // -1 means no limit
+        "save_checkpoint_stride": [
+            3,
+            50
+        ],
+        "keep_last": [
+            3,
+            2
+        ],
+        "run_eval": [
+            true,
+            true
+        ],
+        "adamw": {
+            "lr": 2.0e-4
+        },
+        "reducelronplateau": {
+            "factor": 0.8,
+            "patience": 30,
+            "min_lr": 1.0e-4
+        },
+        "dataloader": {
+            "num_worker": 8,
+            "pin_memory": true
+        },
+        "sampler": {
+            "holistic_shuffle": false,
+            "drop_last": true
+        }
+    }
+}

egs/svc/MultipleContentsSVC/run.sh

@@ -0,0 +1 @@
+../_template/run.sh

egs/svc/README.md

@@ -0,0 +1,34 @@
+# Amphion Singing Voice Conversion (SVC) Recipe
+
+## Quick Start
+
+We provide a **[beginner recipe](MultipleContentsSVC)** to demonstrate how to train a cutting edge SVC model. Specifically, it is also an official implementation of the paper "[Leveraging Content-based Features from Multiple Acoustic Models for Singing Voice Conversion](https://arxiv.org/abs/2310.11160)" (NeurIPS 2023 Workshop on Machine Learning for Audio). Some demos can be seen [here](https://www.zhangxueyao.com/data/MultipleContentsSVC/index.html).
+
+## Supported Model Architectures
+
+The main idea of SVC is to first disentangle the speaker-agnostic representations from the source audio, and then inject the desired speaker information to synthesize the target, which usually utilizes an acoustic decoder and a subsequent waveform synthesizer (vocoder):
+
+<br>
+<div align="center">
+  <img src="../../imgs/svc/pipeline.png" width="70%">
+</div>
+<br>
+
+Until now, Amphion SVC has supported the following features and models:
+
+- **Speaker-agnostic Representations**:
+  - Content Features: Sourcing from [WeNet](https://github.com/wenet-e2e/wenet), [Whisper](https://github.com/openai/whisper), and [ContentVec](https://github.com/auspicious3000/contentvec).
+  - Prosody Features: F0 and energy.
+- **Speaker Embeddings**:
+  - Speaker Look-Up Table.
+  - Reference Encoder (👨‍💻 developing): It can be used for zero-shot SVC.
+- **Acoustic Decoders**:
+  - Diffusion-based models:
+    - **[DiffWaveNetSVC](MultipleContentsSVC)**: The encoder is based on Bidirectional Non-Causal Dilated CNN, which is similar to [WaveNet](https://arxiv.org/pdf/1609.03499.pdf), [DiffWave](https://openreview.net/forum?id=a-xFK8Ymz5J), and [DiffSVC](https://ieeexplore.ieee.org/document/9688219).
+    - **[DiffComoSVC](DiffComoSVC)** (👨‍💻 developing): The diffusion framework is based on [Consistency Model](https://proceedings.mlr.press/v202/song23a.html). It can significantly accelerate the inference process of the diffusion model.
+  - Transformer-based models:
+    - **[TransformerSVC](TransformerSVC)**: Encoder-only and Non-autoregressive Transformer Architecture.
+  - VAE- and Flow-based models:
+    - **[VitsSVC](VitsSVC)**: It is designed as a [VITS](https://arxiv.org/abs/2106.06103)-like model whose textual input is replaced by the content features, which is similar to [so-vits-svc](https://github.com/svc-develop-team/so-vits-svc).
+- **Waveform Synthesizers (Vocoders)**:
+  - The supported vocoders can be seen in [Amphion Vocoder Recipe](../vocoder/README.md).

egs/svc/TransformerSVC/README.md

@@ -0,0 +1,164 @@
+# Transformer for Singing Voice Conversion
+
+This is an implementation of **vanilla transformer encoder**/**conformer** as acoustic model for singing voice conversion.
+
+There are four stages in total:
+
+1. Data preparation
+2. Features extraction
+3. Training
+4. Inference/conversion
+
+> **NOTE:** You need to run every command of this recipe in the `Amphion` root path:
+> ```bash
+> cd Amphion
+> ```
+
+## 1. Data Preparation
+
+### Dataset Download
+
+By default, we utilize the five datasets for training: M4Singer, Opencpop, OpenSinger, SVCC, and VCTK. How to download them is detailed [here](../../datasets/README.md).
+
+### Configuration
+
+Specify the dataset paths in  `exp_config.json`. Note that you can change the `dataset` list to use your preferred datasets.
+
+```json
+    "dataset": [
+        "m4singer",
+        "opencpop",
+        "opensinger",
+        "svcc",
+        "vctk"
+    ],
+    "dataset_path": {
+        // TODO: Fill in your dataset path
+        "m4singer": "[M4Singer dataset path]",
+        "opencpop": "[Opencpop dataset path]",
+        "opensinger": "[OpenSinger dataset path]",
+        "svcc": "[SVCC dataset path]",
+        "vctk": "[VCTK dataset path]"
+    },
+```
+
+## 2. Features Extraction
+
+### Content-based Pretrained Models Download
+
+By default, we utilize the Whisper and ContentVec to extract content features. How to download them is detailed [here](../../../pretrained/README.md).
+
+### Configuration
+
+Specify the dataset path and the output path for saving the processed data and the training model in `exp_config.json`:
+
+```json
+    // TODO: Fill in the output log path. The default value is "Amphion/ckpts/svc"
+    "log_dir": "ckpts/svc",
+    "preprocess": {
+        // TODO: Fill in the output data path. The default value is "Amphion/data"
+        "processed_dir": "data",
+        ...
+    },
+```
+
+### Run
+
+Run the `run.sh` as the preproces stage (set  `--stage 1`).
+
+```bash
+sh egs/svc/TransformerSVC/run.sh --stage 1
+```
+
+> **NOTE:** The `CUDA_VISIBLE_DEVICES` is set as `"0"` in default. You can change it when running `run.sh` by specifying such as `--gpu "1"`.
+
+## 3. Training
+
+### Configuration
+Specify the detailed configuration for transformer block in `exp_config.json`. For key `type`, `conformer` and `transformer` are supported:
+```json
+"model": {
+        ...
+        "transformer":{
+            // 'conformer' or 'transformer'
+            "type": "conformer",
+            "input_dim": 384,
+            "output_dim": 100,
+            "n_heads": 2,
+            "n_layers": 6,
+            "filter_channels":512,
+            "dropout":0.1,
+        }
+    }
+```
+We provide the default hyparameters in the `exp_config.json`. They can work on single NVIDIA-24g GPU. You can adjust them based on you GPU machines.
+
+```json
+"train": {
+        "batch_size": 32,
+        ...
+        "adamw": {
+            "lr": 2.0e-4
+        },
+        ...
+    }
+```
+
+### Run
+
+Run the `run.sh` as the training stage (set  `--stage 2`). Specify a experimental name to run the following command. The tensorboard logs and checkpoints will be saved in `Amphion/ckpts/svc/[YourExptName]`.
+
+```bash
+sh egs/svc/TransformerSVC/run.sh --stage 2 --name [YourExptName]
+```
+
+> **NOTE:** The `CUDA_VISIBLE_DEVICES` is set as `"0"` in default. You can change it when running `run.sh` by specifying such as `--gpu "0,1,2,3"`.
+
+## 4. Inference/Conversion
+
+### Pretrained Vocoder Download
+
+We fine-tune the official BigVGAN pretrained model with over 120 hours singing voice data. The benifits of fine-tuning has been investigated in our paper (see this [demo page](https://www.zhangxueyao.com/data/MultipleContentsSVC/vocoder.html)). The final pretrained singing voice vocoder is released [here](../../../pretrained/README.md#amphion-singing-bigvgan) (called `Amphion Singing BigVGAN`).
+
+### Run
+
+For inference/conversion, you need to specify the following configurations when running `run.sh`:
+
+| Parameters                                          | Description                                                                                                                                | Example                                                                                                                                                                            |
+| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `--infer_expt_dir`                                  | The experimental directory which contains `checkpoint`                                                                                     | `Amphion/ckpts/svc/[YourExptName]`                                                                                                                                                 |
+| `--infer_output_dir`                                | The output directory to save inferred audios.                                                                                              | `Amphion/ckpts/svc/[YourExptName]/result`                                                                                                                                          |
+| `--infer_source_file` or `--infer_source_audio_dir` | The inference source (can be a json file or a dir).                                                                                        | The `infer_source_file` could be `Amphion/data/[YourDataset]/test.json`, and the `infer_source_audio_dir` is a folder which includes several audio files (*.wav, *.mp3 or *.flac). |
+| `--infer_target_speaker`                            | The target speaker you want to convert into. You can refer to `Amphion/ckpts/svc/[YourExptName]/singers.json` to choose a trained speaker. | For opencpop dataset, the speaker name would be `opencpop_female1`.                                                                                                                |
+| `--infer_key_shift`                                 | How many semitones you want to transpose.                                                                                                  | `"autoshfit"` (by default), `3`, `-3`, etc.                                                                                                                                        |
+
+For example, if you want to make `opencpop_female1` sing the songs in the `[Your Audios Folder]`, just run:
+
+```bash
+cd Amphion
+sh egs/svc/TransformerSVC/run.sh --stage 3 --gpu "0" \
+	--infer_expt_dir Amphion/ckpts/svc/[YourExptName] \
+	--infer_output_dir Amphion/ckpts/svc/[YourExptName]/result \
+	--infer_source_audio_dir [Your Audios Folder] \
+	--infer_target_speaker "opencpop_female1" \
+	--infer_key_shift "autoshift"
+```
+
+## Citations
+
+```bibtex
+@inproceedings{transformer,
+  author       = {Ashish Vaswani and
+                  Noam Shazeer and
+                  Niki Parmar and
+                  Jakob Uszkoreit and
+                  Llion Jones and
+                  Aidan N. Gomez and
+                  Lukasz Kaiser and
+                  Illia Polosukhin},
+  title        = {Attention is All you Need},
+  booktitle    = {{NIPS}},
+  pages        = {5998--6008},
+  year         = {2017}
+}
+```

egs/svc/TransformerSVC/exp_config.json

@@ -0,0 +1,108 @@
+{
+    "base_config": "config/transformer.json",
+    "model_type": "TransformerSVC",
+    "dataset": [
+        "m4singer",
+        "opencpop",
+        "opensinger",
+        "svcc",
+        "vctk"
+    ],
+    "dataset_path": {
+        // TODO: Fill in your dataset path
+        "m4singer": "[M4Singer dataset path]",
+        "opencpop": "[Opencpop dataset path]",
+        "opensinger": "[OpenSinger dataset path]",
+        "svcc": "[SVCC dataset path]",
+        "vctk": "[VCTK dataset path]"
+    },
+    // TODO: Fill in the output log path. The default value is "Amphion/ckpts/svc"
+    "log_dir": "ckpts/svc",
+    "preprocess": {
+        // TODO: Fill in the output data path. The default value is "Amphion/data"
+        "processed_dir": "data",
+        // Config for features extraction
+        "extract_mel": true,
+        "extract_pitch": true,
+        "extract_energy": true,
+        "extract_whisper_feature": true,
+        "extract_contentvec_feature": true,
+        "extract_wenet_feature": false,
+        "whisper_batch_size": 30, // decrease it if your GPU is out of memory
+        "contentvec_batch_size": 1,
+        // Fill in the content-based pretrained model's path
+        "contentvec_file": "pretrained/contentvec/checkpoint_best_legacy_500.pt",
+        "wenet_model_path": "pretrained/wenet/20220506_u2pp_conformer_exp/final.pt",
+        "wenet_config": "pretrained/wenet/20220506_u2pp_conformer_exp/train.yaml",
+        "whisper_model": "medium",
+        "whisper_model_path": "pretrained/whisper/medium.pt",
+        // Config for features usage
+        "use_mel": true,
+        "use_min_max_norm_mel": true,
+        "use_frame_pitch": true,
+        "use_frame_energy": true,
+        "use_spkid": true,
+        "use_whisper": true,
+        "use_contentvec": true,
+        "use_wenet": false,
+        "n_mel": 100,
+        "sample_rate": 24000
+    },
+    "model": {
+        "condition_encoder": {
+            // Config for features usage
+            "use_whisper": true,
+            "use_contentvec": true,
+            "use_wenet": false,
+            "whisper_dim": 1024,
+            "contentvec_dim": 256,
+            "wenet_dim": 512,
+            "use_singer_encoder": false,
+            "pitch_min": 50,
+            "pitch_max": 1100
+        },
+        "transformer": {
+            // 'conformer' or 'transformer'
+            "type": "conformer",
+            "input_dim": 384,
+            "output_dim": 100,
+            "n_heads": 2,
+            "n_layers": 6,
+            "filter_channels": 512,
+            "dropout": 0.1,
+        }
+    },
+    "train": {
+        "batch_size": 64,
+        "gradient_accumulation_step": 1,
+        "max_epoch": -1, // -1 means no limit
+        "save_checkpoint_stride": [
+            50,
+            50
+        ],
+        "keep_last": [
+            5,
+            -1
+        ],
+        "run_eval": [
+            false,
+            true
+        ],
+        "adamw": {
+            "lr": 4.0e-4
+        },
+        "reducelronplateau": {
+            "factor": 0.8,
+            "patience": 10,
+            "min_lr": 1.0e-4
+        },
+        "dataloader": {
+            "num_worker": 8,
+            "pin_memory": true
+        },
+        "sampler": {
+            "holistic_shuffle": false,
+            "drop_last": true
+        }
+    }
+}

egs/svc/TransformerSVC/run.sh

@@ -0,0 +1 @@
+../_template/run.sh

egs/svc/VitsSVC/README.md

@@ -0,0 +1,125 @@
+# VITS for Singing Voice Conversion
+
+This is an implementation of VITS as acoustic model for end-to-end singing voice conversion. Adapted from [so-vits-svc](https://github.com/svc-develop-team/so-vits-svc), SoftVC content encoder is used to extract content features from the source audio. These feature vectors are directly fed into VITS without the need for conversion to a text-based intermediate representation.
+
+There are four stages in total:
+
+1. Data preparation
+2. Features extraction
+3. Training
+4. Inference/conversion
+
+> **NOTE:** You need to run every command of this recipe in the `Amphion` root path:
+> ```bash
+> cd Amphion
+> ```
+
+## 1. Data Preparation
+
+### Dataset Download
+
+By default, we utilize the five datasets for training: M4Singer, Opencpop, OpenSinger, SVCC, and VCTK. How to download them is detailed [here](../../datasets/README.md).
+
+### Configuration
+
+Specify the dataset paths in  `exp_config.json`. Note that you can change the `dataset` list to use your preferred datasets.
+
+```json
+    "dataset": [
+        "m4singer",
+        "opencpop",
+        "opensinger",
+        "svcc",
+        "vctk"
+    ],
+    "dataset_path": {
+        // TODO: Fill in your dataset path
+        "m4singer": "[M4Singer dataset path]",
+        "opencpop": "[Opencpop dataset path]",
+        "opensinger": "[OpenSinger dataset path]",
+        "svcc": "[SVCC dataset path]",
+        "vctk": "[VCTK dataset path]"
+    },
+```
+
+## 2. Features Extraction
+
+### Content-based Pretrained Models Download
+
+By default, we utilize ContentVec and Whisper to extract content features. How to download them is detailed [here](../../../pretrained/README.md).
+
+### Configuration
+
+Specify the dataset path and the output path for saving the processed data and the training model in `exp_config.json`:
+
+```json
+    // TODO: Fill in the output log path. The default value is "Amphion/ckpts/svc"
+    "log_dir": "ckpts/svc",
+    "preprocess": {
+        // TODO: Fill in the output data path. The default value is "Amphion/data"
+        "processed_dir": "data",
+        ...
+    },
+```
+
+### Run
+
+Run the `run.sh` as the preproces stage (set  `--stage 1`).
+
+```bash
+sh egs/svc/VitsSVC/run.sh --stage 1
+```
+
+> **NOTE:** The `CUDA_VISIBLE_DEVICES` is set as `"0"` in default. You can change it when running `run.sh` by specifying such as `--gpu "1"`.
+
+## 3. Training
+
+### Configuration
+
+We provide the default hyparameters in the `exp_config.json`. They can work on single NVIDIA-24g GPU. You can adjust them based on you GPU machines.
+
+```json
+"train": {
+        "batch_size": 32,
+        ...
+        "adamw": {
+            "lr": 2.0e-4
+        },
+        ...
+    }
+```
+
+### Run
+
+Run the `run.sh` as the training stage (set  `--stage 2`). Specify a experimental name to run the following command. The tensorboard logs and checkpoints will be saved in `Amphion/ckpts/svc/[YourExptName]`.
+
+```bash
+sh egs/svc/VitsSVC/run.sh --stage 2 --name [YourExptName]
+```
+
+> **NOTE:** The `CUDA_VISIBLE_DEVICES` is set as `"0"` in default. You can change it when running `run.sh` by specifying such as `--gpu "0,1,2,3"`.
+
+## 4. Inference/Conversion
+
+### Run
+
+For inference/conversion, you need to specify the following configurations when running `run.sh`:
+
+| Parameters                                          | Description                                                                                                                                                       | Example                                                                                                                                                                                                  |
+| --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `--infer_expt_dir`                                  | The experimental directory which contains `checkpoint`                                                                                                            | `[Your path to save logs and checkpoints]/[YourExptName]`                                                                                                                                                |
+| `--infer_output_dir`                                | The output directory to save inferred audios.                                                                                                                     | `[Your path to save logs and checkpoints]/[YourExptName]/result`                                                                                                                                         |
+| `--infer_source_file` or `--infer_source_audio_dir` | The inference source (can be a json file or a dir).                                                                                                               | The `infer_source_file` could be `[Your path to save processed data]/[YourDataset]/test.json`, and the `infer_source_audio_dir` is a folder which includes several audio files (*.wav, *.mp3 or *.flac). |
+| `--infer_target_speaker`                            | The target speaker you want to convert into. You can refer to `[Your path to save logs and checkpoints]/[YourExptName]/singers.json` to choose a trained speaker. | For opencpop dataset, the speaker name would be `opencpop_female1`.                                                                                                                                      |
+| `--infer_key_shift`                                 | How many semitones you want to transpose.                                                                                                                         | `"autoshfit"` (by default), `3`, `-3`, etc.                                                                                                                                                              |
+
+For example, if you want to make `opencpop_female1` sing the songs in the `[Your Audios Folder]`, just run:
+
+```bash
+sh egs/svc/VitsSVC/run.sh --stage 3 --gpu "0" \
+	--infer_expt_dir Amphion/ckpts/svc/[YourExptName] \
+	--infer_output_dir Amphion/ckpts/svc/[YourExptName]/result \
+	--infer_source_audio_dir [Your Audios Folder] \
+	--infer_target_speaker "opencpop_female1" \
+	--infer_key_shift "autoshift"
+```

egs/svc/VitsSVC/exp_config.json

@@ -0,0 +1,106 @@
+{
+    "base_config": "config/vitssvc.json",
+    "model_type": "VitsSVC",
+    "dataset": [
+        "m4singer",
+        "opencpop",
+        "opensinger",
+        "svcc",
+        "vctk"
+    ],
+    "dataset_path": {
+        // TODO: Fill in your dataset path
+        "m4singer": "[M4Singer dataset path]",
+        "opencpop": "[Opencpop dataset path]",
+        "opensinger": "[OpenSinger dataset path]",
+        "svcc": "[SVCC dataset path]",
+        "vctk": "[VCTK dataset path]"
+    },
+    "use_custom_dataset": [],
+    // TODO: Fill in the output log path. The default value is "Amphion/ckpts/svc"
+    "log_dir": "ckpts/svc",
+    "preprocess": {
+        // TODO: Fill in the output data path. The default value is "Amphion/data"
+        "processed_dir": "data",
+        
+        "n_mel": 100,
+        "sample_rate": 24000,
+
+        // contentvec
+        "extract_contentvec_feature": true,
+        "contentvec_sample_rate": 16000,
+        "contentvec_batch_size": 1,
+        "contentvec_frameshift": 0.02,
+        // whisper
+        "extract_whisper_feature": true,
+        "whisper_sample_rate": 16000,
+        "whisper_frameshift": 0.01,
+        "whisper_downsample_rate": 2,
+        // wenet
+        "extract_wenet_feature": true,
+        "wenet_downsample_rate": 4,
+        "wenet_frameshift": 0.01,
+        "wenet_sample_rate": 16000,
+        // Fill in the content-based pretrained model's path
+        "contentvec_file": "pretrained/contentvec/checkpoint_best_legacy_500.pt",
+        "wenet_model_path": "pretrained/wenet/20220506_u2pp_conformer_exp/final.pt",
+        "wenet_config": "pretrained/wenet/20220506_u2pp_conformer_exp/train.yaml",
+        "whisper_model": "medium",
+        "whisper_model_path": "pretrained/whisper/medium.pt",
+
+        "use_contentvec": true,
+        "use_whisper": true,
+        "use_wenet": false,
+        
+        // Extract content features using dataloader
+        "pin_memory": true,
+        "num_workers": 8,
+        "content_feature_batch_size": 16,
+
+    },
+    "model": {
+        "condition_encoder": {
+            // Config for features usage
+            "merge_mode": "add",
+            "use_log_loudness": true,
+            "use_contentvec": true,
+            "use_whisper": true,
+            "use_wenet": false,
+            "whisper_dim": 1024,
+            "contentvec_dim": 256,
+            "wenet_dim": 512,
+        },
+        "vits": {
+            "inter_channels": 384,
+            "hidden_channels": 384,
+            "filter_channels": 256,
+            "n_heads": 2,
+            "n_layers": 6,
+            "kernel_size": 3,
+            "p_dropout": 0.1,
+            "n_flow_layer": 4,
+            "n_layers_q": 3,
+            "gin_channels": 256,
+            "n_speakers": 512,
+            "use_spectral_norm": false,
+        },
+        "generator": "nsfhifigan",
+    },
+    "train": {
+        "batch_size": 32,
+        "learning_rate": 2e-4,
+        "gradient_accumulation_step": 1,
+        "max_epoch": -1, // -1 means no limit
+        "save_checkpoint_stride": [
+            3,
+            50
+        ],
+        "keep_last": [
+            3,
+            2
+        ],
+    },
+    "inference": {
+        "batch_size": 1,
+    }
+}

egs/svc/VitsSVC/run.sh

@@ -0,0 +1 @@
+../_template/run.sh

egs/svc/_template/run.sh

@@ -0,0 +1,160 @@
+# Copyright (c) 2023 Amphion.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+######## Build Experiment Environment ###########
+exp_dir=$(cd `dirname $0`; pwd)
+work_dir=$(dirname $(dirname $(dirname $exp_dir)))
+
+export WORK_DIR=$work_dir
+export PYTHONPATH=$work_dir
+export PYTHONIOENCODING=UTF-8
+
+######## Parse the Given Parameters from the Commond ###########
+options=$(getopt -o c:n:s --long gpu:,config:,name:,stage:,resume:,resume_from_ckpt_path:,resume_type:,infer_expt_dir:,infer_output_dir:,infer_source_file:,infer_source_audio_dir:,infer_target_speaker:,infer_key_shift:,infer_vocoder_dir: -- "$@")
+eval set -- "$options"
+
+while true; do
+  case $1 in
+    # Experimental Configuration File
+    -c | --config) shift; exp_config=$1 ; shift ;;
+    # Experimental Name
+    -n | --name) shift; exp_name=$1 ; shift ;;
+    # Running Stage
+    -s | --stage) shift; running_stage=$1 ; shift ;;
+    # Visible GPU machines. The default value is "0".
+    --gpu) shift; gpu=$1 ; shift ;;
+
+    # [Only for Training] Resume configuration
+    --resume) shift; resume=$1 ; shift ;;
+    # [Only for Training] The specific checkpoint path that you want to resume from.
+    --resume_from_ckpt_path) shift; resume_from_ckpt_path=$1 ; shift ;;
+    # [Only for Training] `resume` for loading all the things (including model weights, optimizer, scheduler, and random states). `finetune` for loading only the model weights.
+    --resume_type) shift; resume_type=$1 ; shift ;;
+
+    # [Only for Inference] The experiment dir. The value is like "[Your path to save logs and checkpoints]/[YourExptName]"
+    --infer_expt_dir) shift; infer_expt_dir=$1 ; shift ;;
+    # [Only for Inference] The output dir to save inferred audios. Its default value is "$expt_dir/result"
+    --infer_output_dir) shift; infer_output_dir=$1 ; shift ;;
+    # [Only for Inference] The inference source (can be a json file or a dir). For example, the source_file can be "[Your path to save processed data]/[YourDataset]/test.json", and the source_audio_dir can be "$work_dir/source_audio" which includes several audio files (*.wav, *.mp3 or *.flac).
+    --infer_source_file) shift; infer_source_file=$1 ; shift ;;
+    --infer_source_audio_dir) shift; infer_source_audio_dir=$1 ; shift ;;
+    # [Only for Inference] Specify the target speaker you want to convert into. You can refer to "[Your path to save logs and checkpoints]/[Your Expt Name]/singers.json". In this singer look-up table, you can see the usable speaker names (all the keys of the dictionary). For example, for opencpop dataset, the speaker name would be "opencpop_female1".
+    --infer_target_speaker) shift; infer_target_speaker=$1 ; shift ;;
+    # [Only for Inference] For advanced users, you can modify the trans_key parameters into an integer (which means the semitones you want to transpose). Its default value is "autoshift".
+    --infer_key_shift) shift; infer_key_shift=$1 ; shift ;;
+    # [Only for Inference] The vocoder dir. Its default value is Amphion/pretrained/bigvgan. See Amphion/pretrained/README.md to download the pretrained BigVGAN vocoders.
+    --infer_vocoder_dir) shift; infer_vocoder_dir=$1 ; shift ;;
+
+    --) shift ; break ;;
+    *) echo "Invalid option: $1" exit 1 ;;
+  esac
+done
+
+
+### Value check ###
+if [ -z "$running_stage" ]; then
+    echo "[Error] Please specify the running stage"
+    exit 1
+fi
+
+if [ -z "$exp_config" ]; then
+    exp_config="${exp_dir}"/exp_config.json
+fi
+echo "Exprimental Configuration File: $exp_config"
+
+if [ -z "$gpu" ]; then
+    gpu="0"
+fi
+
+######## Features Extraction ###########
+if [ $running_stage -eq 1 ]; then
+    CUDA_VISIBLE_DEVICES=$gpu python "${work_dir}"/bins/svc/preprocess.py \
+        --config $exp_config \
+        --num_workers 4
+fi
+
+######## Training ###########
+if [ $running_stage -eq 2 ]; then
+    if [ -z "$exp_name" ]; then
+        echo "[Error] Please specify the experiments name"
+        exit 1
+    fi
+    echo "Exprimental Name: $exp_name"
+
+    # add default value
+    if [ -z "$resume_from_ckpt_path" ]; then
+        resume_from_ckpt_path=""
+    fi
+
+    if [ -z "$resume_type" ]; then
+        resume_type="resume"
+    fi
+
+    if [ "$resume" = true ]; then
+        echo "Resume from the existing experiment..."
+        CUDA_VISIBLE_DEVICES="$gpu" accelerate launch "${work_dir}"/bins/svc/train.py \
+            --config "$exp_config" \
+            --exp_name "$exp_name" \
+            --log_level info \
+            --resume \
+            --resume_from_ckpt_path "$resume_from_ckpt_path" \
+            --resume_type "$resume_type"
+    else
+        echo "Start a new experiment..."
+        CUDA_VISIBLE_DEVICES="$gpu" accelerate launch "${work_dir}"/bins/svc/train.py \
+            --config "$exp_config" \
+            --exp_name "$exp_name" \
+            --log_level info
+    fi
+fi
+
+######## Inference/Conversion ###########
+if [ $running_stage -eq 3 ]; then
+    if [ -z "$infer_expt_dir" ]; then
+        echo "[Error] Please specify the experimental directionary. The value is like [Your path to save logs and checkpoints]/[YourExptName]"
+        exit 1
+    fi
+
+    if [ -z "$infer_output_dir" ]; then
+        infer_output_dir="$expt_dir/result"
+    fi
+
+    if [ -z "$infer_source_file" ] && [ -z "$infer_source_audio_dir" ]; then
+        echo "[Error] Please specify the source file/dir. The inference source (can be a json file or a dir). For example, the source_file can be "[Your path to save processed data]/[YourDataset]/test.json", and the source_audio_dir should include several audio files (*.wav, *.mp3 or *.flac)."
+        exit 1
+    fi
+
+    if [ -z "$infer_source_file" ]; then
+        infer_source=$infer_source_audio_dir
+    fi
+
+    if [ -z "$infer_source_audio_dir" ]; then
+        infer_source=$infer_source_file
+    fi
+
+    if [ -z "$infer_target_speaker" ]; then
+        echo "[Error] Please specify the target speaker. You can refer to "[Your path to save logs and checkpoints]/[Your Expt Name]/singers.json". In this singer look-up table, you can see the usable speaker names (all the keys of the dictionary). For example, for opencpop dataset, the speaker name would be "opencpop_female1""
+        exit 1
+    fi
+
+    if [ -z "$infer_key_shift" ]; then
+        infer_key_shift="autoshift"
+    fi
+
+    if [ -z "$infer_vocoder_dir" ]; then
+        infer_vocoder_dir="$work_dir"/pretrained/bigvgan
+        echo "[Warning] You don't specify the infer_vocoder_dir. It is set $infer_vocoder_dir by default. Make sure that you have followed Amphoion/pretrained/README.md to download the pretrained BigVGAN vocoder checkpoint."
+    fi
+
+    CUDA_VISIBLE_DEVICES=$gpu accelerate launch "$work_dir"/bins/svc/inference.py \
+        --config $exp_config \
+        --acoustics_dir $infer_expt_dir \
+        --vocoder_dir $infer_vocoder_dir \
+        --target_singer $infer_target_speaker \
+        --trans_key $infer_key_shift \
+        --source $infer_source \
+        --output_dir $infer_output_dir  \
+        --log_level debug
+fi