| 1 | # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | |
| 15 | import argparse |
| 16 | |
| 17 | import torch |
| 18 | import torch.nn as nn |
| 19 | from conv_stft import STFT |
| 20 | from huggingface_hub import hf_hub_download |
| 21 | from vocos import Vocos |
| 22 | |
| 23 | |
| 24 | opset_version = 17 |
| 25 | |
| 26 | |
| 27 | def get_args(): |
| 28 | parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
| 29 | parser.add_argument( |
| 30 | "--vocoder", |
| 31 | type=str, |
| 32 | default="vocos", |
| 33 | choices=["vocos", "bigvgan"], |
| 34 | help="Vocoder to export", |
| 35 | ) |
| 36 | parser.add_argument( |
| 37 | "--output-path", |
| 38 | type=str, |
| 39 | default="./vocos_vocoder.onnx", |
| 40 | help="Output path", |
| 41 | ) |
| 42 | return parser.parse_args() |
| 43 | |
| 44 | |
| 45 | class ISTFTHead(nn.Module): |
| 46 | def __init__(self, n_fft: int, hop_length: int): |
| 47 | super().__init__() |
| 48 | self.out = None |
| 49 | self.stft = STFT(fft_len=n_fft, win_hop=hop_length, win_len=n_fft) |
| 50 | |
| 51 | def forward(self, x: torch.Tensor): |
| 52 | x = self.out(x).transpose(1, 2) |
| 53 | mag, p = x.chunk(2, dim=1) |
| 54 | mag = torch.exp(mag) |
| 55 | mag = torch.clip(mag, max=1e2) |
| 56 | real = mag * torch.cos(p) |
| 57 | imag = mag * torch.sin(p) |
| 58 | audio = self.stft.inverse(input1=real, input2=imag, input_type="realimag") |
| 59 | return audio |
| 60 | |
| 61 | |
| 62 | class VocosVocoder(nn.Module): |
| 63 | def __init__(self, vocos_vocoder): |
| 64 | super(VocosVocoder, self).__init__() |
| 65 | self.vocos_vocoder = vocos_vocoder |
| 66 | istft_head_out = self.vocos_vocoder.head.out |
| 67 | n_fft = self.vocos_vocoder.head.istft.n_fft |
| 68 | hop_length = self.vocos_vocoder.head.istft.hop_length |
| 69 | istft_head_for_export = ISTFTHead(n_fft, hop_length) |
| 70 | istft_head_for_export.out = istft_head_out |
| 71 | self.vocos_vocoder.head = istft_head_for_export |
| 72 | |
| 73 | def forward(self, mel): |
| 74 | waveform = self.vocos_vocoder.decode(mel) |
| 75 | return waveform |
| 76 | |
| 77 | |
| 78 | def export_VocosVocoder(vocos_vocoder, output_path, verbose): |
| 79 | vocos_vocoder = VocosVocoder(vocos_vocoder).cuda() |
| 80 | vocos_vocoder.eval() |
| 81 | |
| 82 | dummy_batch_size = 8 |
| 83 | dummy_input_length = 500 |
| 84 | |
| 85 | dummy_mel = torch.randn(dummy_batch_size, 100, dummy_input_length).cuda() |
| 86 | |
| 87 | with torch.no_grad(): |
| 88 | dummy_waveform = vocos_vocoder(mel=dummy_mel) |
| 89 | print(dummy_waveform.shape) |
| 90 | |
| 91 | dummy_input = dummy_mel |
| 92 | |
| 93 | torch.onnx.export( |
| 94 | vocos_vocoder, |
| 95 | dummy_input, |
| 96 | output_path, |
| 97 | opset_version=opset_version, |
| 98 | do_constant_folding=True, |
| 99 | input_names=["mel"], |
| 100 | output_names=["waveform"], |
| 101 | dynamic_axes={ |
| 102 | "mel": {0: "batch_size", 2: "input_length"}, |
| 103 | "waveform": {0: "batch_size", 1: "output_length"}, |
| 104 | }, |
| 105 | verbose=verbose, |
| 106 | ) |
| 107 | |
| 108 | print("Exported to {}".format(output_path)) |
| 109 | |
| 110 | |
| 111 | def load_vocoder(vocoder_name="vocos", is_local=False, local_path="", device="cpu", hf_cache_dir=None): |
| 112 | if vocoder_name == "vocos": |
| 113 | # vocoder = Vocos.from_pretrained("charactr/vocos-mel-24khz").to(device) |
| 114 | if is_local: |
| 115 | print(f"Load vocos from local path {local_path}") |
| 116 | config_path = f"{local_path}/config.yaml" |
| 117 | model_path = f"{local_path}/pytorch_model.bin" |
| 118 | else: |
| 119 | print("Download Vocos from huggingface charactr/vocos-mel-24khz") |
| 120 | repo_id = "charactr/vocos-mel-24khz" |
| 121 | config_path = hf_hub_download(repo_id=repo_id, cache_dir=hf_cache_dir, filename="config.yaml") |
| 122 | model_path = hf_hub_download(repo_id=repo_id, cache_dir=hf_cache_dir, filename="pytorch_model.bin") |
| 123 | vocoder = Vocos.from_hparams(config_path) |
| 124 | state_dict = torch.load(model_path, map_location="cpu", weights_only=True) |
| 125 | vocoder.load_state_dict(state_dict) |
| 126 | vocoder = vocoder.eval().to(device) |
| 127 | elif vocoder_name == "bigvgan": |
| 128 | raise NotImplementedError("BigVGAN is not supported yet") |
| 129 | vocoder.remove_weight_norm() |
| 130 | vocoder = vocoder.eval().to(device) |
| 131 | return vocoder |
| 132 | |
| 133 | |
| 134 | if __name__ == "__main__": |
| 135 | args = get_args() |
| 136 | vocoder = load_vocoder(vocoder_name=args.vocoder, device="cpu", hf_cache_dir=None) |
| 137 | if args.vocoder == "vocos": |
| 138 | export_VocosVocoder(vocoder, args.output_path, verbose=False) |
| 139 |