| 1 | """ADAPTIVE BATCH SIZE""" |
| 2 | |
| 3 | print("Adaptive batch size: using grouping batch sampler, frames_per_gpu fixed fed in") |
| 4 | print(" -> least padding, gather wavs with accumulated frames in a batch\n") |
| 5 | |
| 6 | # data |
| 7 | total_hours = 95282 |
| 8 | mel_hop_length = 256 |
| 9 | mel_sampling_rate = 24000 |
| 10 | |
| 11 | # target |
| 12 | wanted_max_updates = 1200000 |
| 13 | |
| 14 | # train params |
| 15 | gpus = 8 |
| 16 | frames_per_gpu = 38400 # 8 * 38400 = 307200 |
| 17 | grad_accum = 1 |
| 18 | |
| 19 | # intermediate |
| 20 | mini_batch_frames = frames_per_gpu * grad_accum * gpus |
| 21 | mini_batch_hours = mini_batch_frames * mel_hop_length / mel_sampling_rate / 3600 |
| 22 | updates_per_epoch = total_hours / mini_batch_hours |
| 23 | # steps_per_epoch = updates_per_epoch * grad_accum |
| 24 | |
| 25 | # result |
| 26 | epochs = wanted_max_updates / updates_per_epoch |
| 27 | print(f"epochs should be set to: {epochs:.0f} ({epochs / grad_accum:.1f} x gd_acum {grad_accum})") |
| 28 | print(f"progress_bar should show approx. 0/{updates_per_epoch:.0f} updates") |
| 29 | # print(f" or approx. 0/{steps_per_epoch:.0f} steps") |
| 30 | |
| 31 | # others |
| 32 | print(f"total {total_hours:.0f} hours") |
| 33 | print(f"mini-batch of {mini_batch_frames:.0f} frames, {mini_batch_hours:.2f} hours per mini-batch") |
| 34 |