Update app.py

app.py

@@ -10,11 +10,13 @@ import numpy as np
 from pydub import AudioSegment
 
 # Load model and configuration
-device = 'cuda'
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
 dit_checkpoint_path, dit_config_path = load_custom_model_from_hf("Plachta/Seed-VC",
                                                 "DiT_seed_v2_uvit_whisper_small_wavenet_bigvgan_pruned.pth",
                                                 "config_dit_mel_seed_uvit_whisper_small_wavenet.yml")
+# dit_checkpoint_path = "E:/DiT_epoch_00018_step_801000.pth"
+# dit_config_path = "configs/config_dit_mel_seed_uvit_whisper_small_encoder_wavenet.yml"
 config = yaml.safe_load(open(dit_config_path, 'r'))
 model_params = recursive_munch(config['model_params'])
 model = build_model(model_params, stage='DiT')
@@ -46,6 +48,19 @@ bigvgan_model = bigvgan.BigVGAN.from_pretrained('nvidia/bigvgan_v2_22khz_80band_
 bigvgan_model.remove_weight_norm()
 bigvgan_model = bigvgan_model.eval().to(device)
 
+ckpt_path, config_path = load_custom_model_from_hf("Plachta/FAcodec", 'pytorch_model.bin', 'config.yml')
+
+codec_config = yaml.safe_load(open(config_path))
+codec_model_params = recursive_munch(codec_config['model_params'])
+codec_encoder = build_model(codec_model_params, stage="codec")
+
+ckpt_params = torch.load(ckpt_path, map_location="cpu")
+
+for key in codec_encoder:
+    codec_encoder[key].load_state_dict(ckpt_params[key], strict=False)
+_ = [codec_encoder[key].eval() for key in codec_encoder]
+_ = [codec_encoder[key].to(device) for key in codec_encoder]
+
 # whisper
 from transformers import AutoFeatureExtractor, WhisperModel
 
@@ -119,16 +134,12 @@ def adjust_f0_semitones(f0_sequence, n_semitones):
 def crossfade(chunk1, chunk2, overlap):
     fade_out = np.cos(np.linspace(0, np.pi / 2, overlap)) ** 2
     fade_in = np.cos(np.linspace(np.pi / 2, 0, overlap)) ** 2
-    if len(chunk2) < overlap:
-        chunk2[:overlap] = chunk2[:overlap] * fade_in[:len(chunk2)] + (chunk1[-overlap:] * fade_out)[:len(chunk2)]
-    else:
-        chunk2[:overlap] = chunk2[:overlap] * fade_in + chunk1[-overlap:] * fade_out
+    chunk2[:overlap] = chunk2[:overlap] * fade_in + chunk1[-overlap:] * fade_out
     return chunk2
 
 # streaming and chunk processing related params
-overlap_frame_len = 16
 bitrate = "320k"
-
+overlap_frame_len = 16
 @spaces.GPU
 @torch.no_grad()
 @torch.inference_mode()
@@ -232,8 +243,8 @@ def voice_conversion(source, target, diffusion_steps, length_adjust, inference_c
     style2 = campplus_model(feat2.unsqueeze(0))
 
     if f0_condition:
-        F0_ori = rmvpe.infer_from_audio(ref_waves_16k[0], thred=0.03)
-        F0_alt = rmvpe.infer_from_audio(converted_waves_16k[0], thred=0.03)
+        F0_ori = rmvpe.infer_from_audio(ref_waves_16k[0], thred=0.5)
+        F0_alt = rmvpe.infer_from_audio(converted_waves_16k[0], thred=0.5)
 
         F0_ori = torch.from_numpy(F0_ori).to(device)[None]
         F0_alt = torch.from_numpy(F0_alt).to(device)[None]
@@ -272,7 +283,7 @@ def voice_conversion(source, target, diffusion_steps, length_adjust, inference_c
         chunk_cond = cond[:, processed_frames:processed_frames + max_source_window]
         is_last_chunk = processed_frames + max_source_window >= cond.size(1)
         cat_condition = torch.cat([prompt_condition, chunk_cond], dim=1)
-        with torch.autocast(device_type=device.type, dtype=torch.float16):
+        with torch.autocast(device_type='cuda', dtype=torch.float16):
             # Voice Conversion
             vc_target = inference_module.cfm.inference(cat_condition,
                                                        torch.LongTensor([cat_condition.size(1)]).to(mel2.device),
@@ -326,7 +337,7 @@ def voice_conversion(source, target, diffusion_steps, length_adjust, inference_c
 
 
 if __name__ == "__main__":
-    description = ("Zero-shot voice conversion with in-context learning. For local deployment please check [GitHub repository](https://github.com/Plachtaa/seed-vc) "
+    description = ("State-of-the-Art zero-shot voice conversion/singing voice conversion. For local deployment please check [GitHub repository](https://github.com/Plachtaa/seed-vc) "
                    "for details and updates.<br>Note that any reference audio will be forcefully clipped to 25s if beyond this length.<br> "
                    "If total duration of source and reference audio exceeds 30s, source audio will be processed in chunks.<br> "
                    "无需训练的 zero-shot 语音/歌声转换模型，若需本地部署查看[GitHub页面](https://github.com/Plachtaa/seed-vc)<br>"
@@ -334,7 +345,7 @@ if __name__ == "__main__":
     inputs = [
         gr.Audio(type="filepath", label="Source Audio / 源音频"),
         gr.Audio(type="filepath", label="Reference Audio / 参考音频"),
-        gr.Slider(minimum=1, maximum=200, value=10, step=1, label="Diffusion Steps / 扩散步数", info="10 by default, 50~100 for best quality / 默认为 10，50~100 为最佳质量"),
+        gr.Slider(minimum=1, maximum=200, value=25, step=1, label="Diffusion Steps / 扩散步数", info="25 by default, 50~100 for best quality / 默认为 25，50~100 为最佳质量"),
         gr.Slider(minimum=0.5, maximum=2.0, step=0.1, value=1.0, label="Length Adjust / 长度调整", info="<1.0 for speed-up speech, >1.0 for slow-down speech / <1.0 加速语速，>1.0 减慢语速"),
         gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=0.7, label="Inference CFG Rate", info="has subtle influence / 有微小影响"),
         gr.Checkbox(label="Use F0 conditioned model / 启用F0输入", value=False, info="Must set to true for singing voice conversion / 歌声转换时必须勾选"),
@@ -344,11 +355,11 @@ if __name__ == "__main__":
     ]
 
     examples = [["examples/source/yae_0.wav", "examples/reference/dingzhen_0.wav", 25, 1.0, 0.7, False, True, 0],
-                ["examples/source/jay_0.wav", "examples/reference/azuma_0.wav", 25, 1.0, 0.7, True, True, 0],
+                ["examples/source/jay_0.wav", "examples/reference/azuma_0.wav", 25, 1.0, 0.7, False, True, 0],
                 ["examples/source/Wiz Khalifa,Charlie Puth - See You Again [vocals]_[cut_28sec].wav",
-                 "examples/reference/teio_0.wav", 25, 1.0, 0.7, True, False, 0],
+                 "examples/reference/kobe_0.wav", 50, 1.0, 0.7, True, False, -6],
                 ["examples/source/TECHNOPOLIS - 2085 [vocals]_[cut_14sec].wav",
-                 "examples/reference/trump_0.wav", 25, 1.0, 0.7, True, False, -12],
+                 "examples/reference/trump_0.wav", 50, 1.0, 0.7, True, False, -12],
                 ]
 
     outputs = [gr.Audio(label="Stream Output Audio / 流式输出", streaming=True, format='mp3'),