| 1 | from __future__ import annotations |
| 2 | |
| 3 | import gc |
| 4 | import math |
| 5 | import os |
| 6 | |
| 7 | import torch |
| 8 | import torchaudio |
| 9 | import wandb |
| 10 | from accelerate import Accelerator |
| 11 | from accelerate.utils import DistributedDataParallelKwargs |
| 12 | from ema_pytorch import EMA |
| 13 | from torch.optim import AdamW |
| 14 | from torch.optim.lr_scheduler import LinearLR, SequentialLR |
| 15 | from torch.utils.data import DataLoader, Dataset, SequentialSampler |
| 16 | from tqdm import tqdm |
| 17 | |
| 18 | from f5_tts.model import CFM |
| 19 | from f5_tts.model.dataset import DynamicBatchSampler, collate_fn |
| 20 | from f5_tts.model.utils import default, exists |
| 21 | |
| 22 | |
| 23 | # trainer |
| 24 | |
| 25 | |
| 26 | class Trainer: |
| 27 | def __init__( |
| 28 | self, |
| 29 | model: CFM, |
| 30 | epochs, |
| 31 | learning_rate, |
| 32 | num_warmup_updates=20000, |
| 33 | save_per_updates=1000, |
| 34 | keep_last_n_checkpoints: int = -1, # -1 to keep all, 0 to not save intermediate, > 0 to keep last N checkpoints |
| 35 | checkpoint_path=None, |
| 36 | batch_size_per_gpu=32, |
| 37 | batch_size_type: str = "sample", |
| 38 | max_samples=32, |
| 39 | grad_accumulation_steps=1, |
| 40 | max_grad_norm=1.0, |
| 41 | noise_scheduler: str | None = None, |
| 42 | duration_predictor: torch.nn.Module | None = None, |
| 43 | logger: str | None = "wandb", # "wandb" | "tensorboard" | None |
| 44 | wandb_project="test_f5-tts", |
| 45 | wandb_run_name="test_run", |
| 46 | wandb_resume_id: str = None, |
| 47 | log_samples: bool = False, |
| 48 | last_per_updates=None, |
| 49 | accelerate_kwargs: dict = dict(), |
| 50 | ema_kwargs: dict = dict(), |
| 51 | bnb_optimizer: bool = False, |
| 52 | mel_spec_type: str = "vocos", # "vocos" | "bigvgan" |
| 53 | is_local_vocoder: bool = False, # use local path vocoder |
| 54 | local_vocoder_path: str = "", # local vocoder path |
| 55 | model_cfg_dict: dict = dict(), # training config |
| 56 | ): |
| 57 | ddp_kwargs = DistributedDataParallelKwargs(find_unused_parameters=True) |
| 58 | |
| 59 | if logger == "wandb" and not wandb.api.api_key: |
| 60 | logger = None |
| 61 | self.log_samples = log_samples |
| 62 | |
| 63 | self.accelerator = Accelerator( |
| 64 | log_with=logger if logger == "wandb" else None, |
| 65 | kwargs_handlers=[ddp_kwargs], |
| 66 | gradient_accumulation_steps=grad_accumulation_steps, |
| 67 | **accelerate_kwargs, |
| 68 | ) |
| 69 | |
| 70 | self.logger = logger |
| 71 | if self.logger == "wandb": |
| 72 | if exists(wandb_resume_id): |
| 73 | init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name, "id": wandb_resume_id}} |
| 74 | else: |
| 75 | init_kwargs = {"wandb": {"resume": "allow", "name": wandb_run_name}} |
| 76 | |
| 77 | if not model_cfg_dict: |
| 78 | model_cfg_dict = { |
| 79 | "epochs": epochs, |
| 80 | "learning_rate": learning_rate, |
| 81 | "num_warmup_updates": num_warmup_updates, |
| 82 | "batch_size_per_gpu": batch_size_per_gpu, |
| 83 | "batch_size_type": batch_size_type, |
| 84 | "max_samples": max_samples, |
| 85 | "grad_accumulation_steps": grad_accumulation_steps, |
| 86 | "max_grad_norm": max_grad_norm, |
| 87 | "noise_scheduler": noise_scheduler, |
| 88 | "bnb_optimizer": bnb_optimizer, |
| 89 | } |
| 90 | model_cfg_dict["gpus"] = self.accelerator.num_processes |
| 91 | self.accelerator.init_trackers( |
| 92 | project_name=wandb_project, |
| 93 | init_kwargs=init_kwargs, |
| 94 | config=model_cfg_dict, |
| 95 | ) |
| 96 | |
| 97 | elif self.logger == "tensorboard": |
| 98 | from torch.utils.tensorboard import SummaryWriter |
| 99 | |
| 100 | self.writer = None |
| 101 | if self.accelerator.is_main_process: |
| 102 | self.writer = SummaryWriter(log_dir=f"runs/{wandb_run_name}") |
| 103 | |
| 104 | self.model = model |
| 105 | |
| 106 | if self.is_main: |
| 107 | self.ema_model = EMA(model, include_online_model=False, **ema_kwargs) |
| 108 | self.ema_model.to(self.accelerator.device) |
| 109 | |
| 110 | print(f"Using logger: {logger}") |
| 111 | if grad_accumulation_steps > 1: |
| 112 | print( |
| 113 | "Gradient accumulation checkpointing with per_updates now, old logic per_steps used with before f992c4e" |
| 114 | ) |
| 115 | |
| 116 | self.epochs = epochs |
| 117 | self.num_warmup_updates = num_warmup_updates |
| 118 | self.save_per_updates = save_per_updates |
| 119 | self.keep_last_n_checkpoints = keep_last_n_checkpoints |
| 120 | self.last_per_updates = default(last_per_updates, save_per_updates) |
| 121 | self.checkpoint_path = default(checkpoint_path, "ckpts/test_f5-tts") |
| 122 | |
| 123 | self.batch_size_per_gpu = batch_size_per_gpu |
| 124 | self.batch_size_type = batch_size_type |
| 125 | self.max_samples = max_samples |
| 126 | self.grad_accumulation_steps = grad_accumulation_steps |
| 127 | self.max_grad_norm = max_grad_norm |
| 128 | |
| 129 | # mel vocoder config |
| 130 | self.vocoder_name = mel_spec_type |
| 131 | self.is_local_vocoder = is_local_vocoder |
| 132 | self.local_vocoder_path = local_vocoder_path |
| 133 | |
| 134 | self.noise_scheduler = noise_scheduler |
| 135 | |
| 136 | self.duration_predictor = duration_predictor |
| 137 | |
| 138 | if bnb_optimizer: |
| 139 | import bitsandbytes as bnb |
| 140 | |
| 141 | self.optimizer = bnb.optim.AdamW8bit(model.parameters(), lr=learning_rate) |
| 142 | else: |
| 143 | self.optimizer = AdamW(model.parameters(), lr=learning_rate, fused=True) |
| 144 | self.model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer) |
| 145 | |
| 146 | @property |
| 147 | def is_main(self): |
| 148 | return self.accelerator.is_main_process |
| 149 | |
| 150 | def save_checkpoint(self, update, last=False): |
| 151 | self.accelerator.wait_for_everyone() |
| 152 | if self.is_main: |
| 153 | checkpoint = dict( |
| 154 | model_state_dict=self.accelerator.unwrap_model(self.model).state_dict(), |
| 155 | optimizer_state_dict=self.optimizer.state_dict(), |
| 156 | ema_model_state_dict=self.ema_model.state_dict(), |
| 157 | scheduler_state_dict=self.scheduler.state_dict(), |
| 158 | update=update, |
| 159 | ) |
| 160 | if not os.path.exists(self.checkpoint_path): |
| 161 | os.makedirs(self.checkpoint_path) |
| 162 | if last: |
| 163 | self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_last.pt") |
| 164 | print(f"Saved last checkpoint at update {update}") |
| 165 | else: |
| 166 | if self.keep_last_n_checkpoints == 0: |
| 167 | return |
| 168 | self.accelerator.save(checkpoint, f"{self.checkpoint_path}/model_{update}.pt") |
| 169 | if self.keep_last_n_checkpoints > 0: |
| 170 | # Updated logic to exclude pretrained model from rotation |
| 171 | checkpoints = [ |
| 172 | f |
| 173 | for f in os.listdir(self.checkpoint_path) |
| 174 | if f.startswith("model_") |
| 175 | and not f.startswith("pretrained_") # Exclude pretrained models |
| 176 | and f.endswith(".pt") |
| 177 | and f != "model_last.pt" |
| 178 | ] |
| 179 | checkpoints.sort(key=lambda x: int(x.split("_")[1].split(".")[0])) |
| 180 | while len(checkpoints) > self.keep_last_n_checkpoints: |
| 181 | oldest_checkpoint = checkpoints.pop(0) |
| 182 | os.remove(os.path.join(self.checkpoint_path, oldest_checkpoint)) |
| 183 | print(f"Removed old checkpoint: {oldest_checkpoint}") |
| 184 | |
| 185 | def load_checkpoint(self): |
| 186 | if ( |
| 187 | not exists(self.checkpoint_path) |
| 188 | or not os.path.exists(self.checkpoint_path) |
| 189 | or not any(filename.endswith((".pt", ".safetensors")) for filename in os.listdir(self.checkpoint_path)) |
| 190 | ): |
| 191 | return 0 |
| 192 | |
| 193 | self.accelerator.wait_for_everyone() |
| 194 | if "model_last.pt" in os.listdir(self.checkpoint_path): |
| 195 | latest_checkpoint = "model_last.pt" |
| 196 | else: |
| 197 | # Updated to consider pretrained models for loading but prioritize training checkpoints |
| 198 | all_checkpoints = [ |
| 199 | f |
| 200 | for f in os.listdir(self.checkpoint_path) |
| 201 | if (f.startswith("model_") or f.startswith("pretrained_")) and f.endswith((".pt", ".safetensors")) |
| 202 | ] |
| 203 | |
| 204 | # First try to find regular training checkpoints |
| 205 | training_checkpoints = [f for f in all_checkpoints if f.startswith("model_") and f != "model_last.pt"] |
| 206 | if training_checkpoints: |
| 207 | latest_checkpoint = sorted( |
| 208 | training_checkpoints, |
| 209 | key=lambda x: int("".join(filter(str.isdigit, x))), |
| 210 | )[-1] |
| 211 | else: |
| 212 | # If no training checkpoints, use pretrained model |
| 213 | latest_checkpoint = next(f for f in all_checkpoints if f.startswith("pretrained_")) |
| 214 | |
| 215 | if latest_checkpoint.endswith(".safetensors"): # always a pretrained checkpoint |
| 216 | from safetensors.torch import load_file |
| 217 | |
| 218 | checkpoint = load_file(f"{self.checkpoint_path}/{latest_checkpoint}", device="cpu") |
| 219 | checkpoint = {"ema_model_state_dict": checkpoint} |
| 220 | elif latest_checkpoint.endswith(".pt"): |
| 221 | # checkpoint = torch.load(f"{self.checkpoint_path}/{latest_checkpoint}", map_location=self.accelerator.device) # rather use accelerator.load_state ಥ_ಥ |
| 222 | checkpoint = torch.load( |
| 223 | f"{self.checkpoint_path}/{latest_checkpoint}", weights_only=True, map_location="cpu" |
| 224 | ) |
| 225 | |
| 226 | # patch for backward compatibility, 305e3ea |
| 227 | for key in ["ema_model.mel_spec.mel_stft.mel_scale.fb", "ema_model.mel_spec.mel_stft.spectrogram.window"]: |
| 228 | if key in checkpoint["ema_model_state_dict"]: |
| 229 | del checkpoint["ema_model_state_dict"][key] |
| 230 | |
| 231 | if self.is_main: |
| 232 | self.ema_model.load_state_dict(checkpoint["ema_model_state_dict"]) |
| 233 | |
| 234 | if "update" in checkpoint or "step" in checkpoint: |
| 235 | # patch for backward compatibility, with before f992c4e |
| 236 | if "step" in checkpoint: |
| 237 | checkpoint["update"] = checkpoint["step"] // self.grad_accumulation_steps |
| 238 | if self.grad_accumulation_steps > 1 and self.is_main: |
| 239 | print( |
| 240 | "F5-TTS WARNING: Loading checkpoint saved with per_steps logic (before f992c4e), will convert to per_updates according to grad_accumulation_steps setting, may have unexpected behaviour." |
| 241 | ) |
| 242 | # patch for backward compatibility, 305e3ea |
| 243 | for key in ["mel_spec.mel_stft.mel_scale.fb", "mel_spec.mel_stft.spectrogram.window"]: |
| 244 | if key in checkpoint["model_state_dict"]: |
| 245 | del checkpoint["model_state_dict"][key] |
| 246 | |
| 247 | self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"]) |
| 248 | self.optimizer.load_state_dict(checkpoint["optimizer_state_dict"]) |
| 249 | if self.scheduler: |
| 250 | self.scheduler.load_state_dict(checkpoint["scheduler_state_dict"]) |
| 251 | update = checkpoint["update"] |
| 252 | else: |
| 253 | checkpoint["model_state_dict"] = { |
| 254 | k.replace("ema_model.", ""): v |
| 255 | for k, v in checkpoint["ema_model_state_dict"].items() |
| 256 | if k not in ["initted", "update", "step"] |
| 257 | } |
| 258 | self.accelerator.unwrap_model(self.model).load_state_dict(checkpoint["model_state_dict"]) |
| 259 | update = 0 |
| 260 | |
| 261 | del checkpoint |
| 262 | gc.collect() |
| 263 | return update |
| 264 | |
| 265 | def train(self, train_dataset: Dataset, num_workers=16, resumable_with_seed: int = None): |
| 266 | if self.log_samples: |
| 267 | from f5_tts.infer.utils_infer import cfg_strength, load_vocoder, nfe_step, sway_sampling_coef |
| 268 | |
| 269 | vocoder = load_vocoder( |
| 270 | vocoder_name=self.vocoder_name, is_local=self.is_local_vocoder, local_path=self.local_vocoder_path |
| 271 | ) |
| 272 | target_sample_rate = self.accelerator.unwrap_model(self.model).mel_spec.target_sample_rate |
| 273 | log_samples_path = f"{self.checkpoint_path}/samples" |
| 274 | os.makedirs(log_samples_path, exist_ok=True) |
| 275 | |
| 276 | if exists(resumable_with_seed): |
| 277 | generator = torch.Generator() |
| 278 | generator.manual_seed(resumable_with_seed) |
| 279 | else: |
| 280 | generator = None |
| 281 | |
| 282 | if self.batch_size_type == "sample": |
| 283 | train_dataloader = DataLoader( |
| 284 | train_dataset, |
| 285 | collate_fn=collate_fn, |
| 286 | num_workers=num_workers, |
| 287 | pin_memory=True, |
| 288 | persistent_workers=True, |
| 289 | batch_size=self.batch_size_per_gpu, |
| 290 | shuffle=True, |
| 291 | generator=generator, |
| 292 | ) |
| 293 | elif self.batch_size_type == "frame": |
| 294 | self.accelerator.even_batches = False |
| 295 | sampler = SequentialSampler(train_dataset) |
| 296 | batch_sampler = DynamicBatchSampler( |
| 297 | sampler, |
| 298 | self.batch_size_per_gpu, |
| 299 | max_samples=self.max_samples, |
| 300 | random_seed=resumable_with_seed, # This enables reproducible shuffling |
| 301 | drop_residual=False, |
| 302 | ) |
| 303 | train_dataloader = DataLoader( |
| 304 | train_dataset, |
| 305 | collate_fn=collate_fn, |
| 306 | num_workers=num_workers, |
| 307 | pin_memory=True, |
| 308 | persistent_workers=True, |
| 309 | batch_sampler=batch_sampler, |
| 310 | ) |
| 311 | else: |
| 312 | raise ValueError(f"batch_size_type must be either 'sample' or 'frame', but received {self.batch_size_type}") |
| 313 | |
| 314 | # accelerator.prepare() dispatches batches to devices; |
| 315 | # which means the length of dataloader calculated before, should consider the number of devices |
| 316 | warmup_updates = ( |
| 317 | self.num_warmup_updates * self.accelerator.num_processes |
| 318 | ) # consider a fixed warmup steps while using accelerate multi-gpu ddp |
| 319 | # otherwise by default with split_batches=False, warmup steps change with num_processes |
| 320 | total_updates = math.ceil(len(train_dataloader) / self.grad_accumulation_steps) * self.epochs |
| 321 | decay_updates = total_updates - warmup_updates |
| 322 | warmup_scheduler = LinearLR(self.optimizer, start_factor=1e-8, end_factor=1.0, total_iters=warmup_updates) |
| 323 | decay_scheduler = LinearLR(self.optimizer, start_factor=1.0, end_factor=1e-8, total_iters=decay_updates) |
| 324 | self.scheduler = SequentialLR( |
| 325 | self.optimizer, schedulers=[warmup_scheduler, decay_scheduler], milestones=[warmup_updates] |
| 326 | ) |
| 327 | train_dataloader, self.scheduler = self.accelerator.prepare( |
| 328 | train_dataloader, self.scheduler |
| 329 | ) # actual multi_gpu updates = single_gpu updates / gpu nums |
| 330 | start_update = self.load_checkpoint() |
| 331 | global_update = start_update |
| 332 | |
| 333 | if exists(resumable_with_seed): |
| 334 | orig_epoch_step = len(train_dataloader) |
| 335 | start_step = start_update * self.grad_accumulation_steps |
| 336 | skipped_epoch = int(start_step // orig_epoch_step) |
| 337 | skipped_batch = start_step % orig_epoch_step |
| 338 | skipped_dataloader = self.accelerator.skip_first_batches(train_dataloader, num_batches=skipped_batch) |
| 339 | else: |
| 340 | skipped_epoch = 0 |
| 341 | |
| 342 | for epoch in range(skipped_epoch, self.epochs): |
| 343 | self.model.train() |
| 344 | if exists(resumable_with_seed) and epoch == skipped_epoch: |
| 345 | progress_bar_initial = math.ceil(skipped_batch / self.grad_accumulation_steps) |
| 346 | current_dataloader = skipped_dataloader |
| 347 | else: |
| 348 | progress_bar_initial = 0 |
| 349 | current_dataloader = train_dataloader |
| 350 | |
| 351 | # Set epoch for the batch sampler if it exists |
| 352 | if hasattr(train_dataloader, "batch_sampler") and hasattr(train_dataloader.batch_sampler, "set_epoch"): |
| 353 | train_dataloader.batch_sampler.set_epoch(epoch) |
| 354 | |
| 355 | progress_bar = tqdm( |
| 356 | range(math.ceil(len(train_dataloader) / self.grad_accumulation_steps)), |
| 357 | desc=f"Epoch {epoch + 1}/{self.epochs}", |
| 358 | unit="update", |
| 359 | disable=not self.accelerator.is_local_main_process, |
| 360 | initial=progress_bar_initial, |
| 361 | ) |
| 362 | |
| 363 | for batch in current_dataloader: |
| 364 | with self.accelerator.accumulate(self.model): |
| 365 | text_inputs = batch["text"] |
| 366 | mel_spec = batch["mel"].permute(0, 2, 1) |
| 367 | mel_lengths = batch["mel_lengths"] |
| 368 | |
| 369 | # TODO. add duration predictor training |
| 370 | if self.duration_predictor is not None and self.accelerator.is_local_main_process: |
| 371 | dur_loss = self.duration_predictor(mel_spec, lens=batch.get("durations")) |
| 372 | self.accelerator.log({"duration loss": dur_loss.item()}, step=global_update) |
| 373 | |
| 374 | loss, cond, pred = self.model( |
| 375 | mel_spec, text=text_inputs, lens=mel_lengths, noise_scheduler=self.noise_scheduler |
| 376 | ) |
| 377 | self.accelerator.backward(loss) |
| 378 | |
| 379 | if self.max_grad_norm > 0 and self.accelerator.sync_gradients: |
| 380 | self.accelerator.clip_grad_norm_(self.model.parameters(), self.max_grad_norm) |
| 381 | |
| 382 | self.optimizer.step() |
| 383 | self.scheduler.step() |
| 384 | self.optimizer.zero_grad() |
| 385 | |
| 386 | if self.accelerator.sync_gradients: |
| 387 | if self.is_main: |
| 388 | self.ema_model.update() |
| 389 | |
| 390 | global_update += 1 |
| 391 | progress_bar.update(1) |
| 392 | progress_bar.set_postfix(update=str(global_update), loss=loss.item()) |
| 393 | |
| 394 | if self.accelerator.is_local_main_process: |
| 395 | self.accelerator.log( |
| 396 | {"loss": loss.item(), "lr": self.scheduler.get_last_lr()[0]}, step=global_update |
| 397 | ) |
| 398 | if self.logger == "tensorboard" and self.accelerator.is_main_process: |
| 399 | self.writer.add_scalar("loss", loss.item(), global_update) |
| 400 | self.writer.add_scalar("lr", self.scheduler.get_last_lr()[0], global_update) |
| 401 | |
| 402 | if global_update % self.last_per_updates == 0 and self.accelerator.sync_gradients: |
| 403 | self.save_checkpoint(global_update, last=True) |
| 404 | |
| 405 | if global_update % self.save_per_updates == 0 and self.accelerator.sync_gradients: |
| 406 | self.save_checkpoint(global_update) |
| 407 | |
| 408 | if self.log_samples and self.accelerator.is_local_main_process: |
| 409 | ref_audio_len = mel_lengths[0] |
| 410 | infer_text = [ |
| 411 | text_inputs[0] + ([" "] if isinstance(text_inputs[0], list) else " ") + text_inputs[0] |
| 412 | ] |
| 413 | with torch.inference_mode(), self.accelerator.autocast(): |
| 414 | generated, _ = self.accelerator.unwrap_model(self.model).sample( |
| 415 | cond=mel_spec[0][:ref_audio_len].unsqueeze(0), |
| 416 | text=infer_text, |
| 417 | duration=ref_audio_len * 2, |
| 418 | steps=nfe_step, |
| 419 | cfg_strength=cfg_strength, |
| 420 | sway_sampling_coef=sway_sampling_coef, |
| 421 | ) |
| 422 | generated = generated.to(torch.float32) |
| 423 | gen_mel_spec = generated[:, ref_audio_len:, :].permute(0, 2, 1).to(self.accelerator.device) |
| 424 | ref_mel_spec = batch["mel"][0, :, :ref_audio_len].unsqueeze(0) |
| 425 | if self.vocoder_name == "vocos": |
| 426 | gen_audio = vocoder.decode(gen_mel_spec).cpu() |
| 427 | ref_audio = vocoder.decode(ref_mel_spec).cpu() |
| 428 | elif self.vocoder_name == "bigvgan": |
| 429 | gen_audio = vocoder(gen_mel_spec).squeeze(0).cpu() |
| 430 | ref_audio = vocoder(ref_mel_spec).squeeze(0).cpu() |
| 431 | |
| 432 | torchaudio.save( |
| 433 | f"{log_samples_path}/update_{global_update}_gen.wav", gen_audio, target_sample_rate |
| 434 | ) |
| 435 | torchaudio.save( |
| 436 | f"{log_samples_path}/update_{global_update}_ref.wav", ref_audio, target_sample_rate |
| 437 | ) |
| 438 | self.model.train() |
| 439 | |
| 440 | self.save_checkpoint(global_update, last=True) |
| 441 | |
| 442 | self.accelerator.end_training() |
| 443 |