diff --git "a/trainer_0_log.txt" "b/trainer_0_log.txt" new file mode 100644--- /dev/null +++ "b/trainer_0_log.txt" @@ -0,0 +1,38563 @@ + > Training Environment: + | > Backend: Torch + | > Mixed precision: True + | > Precision: fp16 + | > Current device: 0 + | > Num. of GPUs: 1 + | > Num. of CPUs: 4 + | > Num. of Torch Threads: 2 + | > Torch seed: 54321 + | > Torch CUDNN: True + | > Torch CUDNN deterministic: False + | > Torch CUDNN benchmark: True + | > Torch TF32 MatMul: False + > Start Tensorboard: tensorboard --logdir=/home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > Model has 86444204 parameters + + > EPOCH: 0/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.004019610425259204 (+0) + | > avg_loss_disc: 5.960158490120096 (+0) + | > avg_loss_disc_real_0: 0.9988338921932464 (+0) + | > avg_loss_disc_real_1: 0.9593739661764591 (+0) + | > avg_loss_disc_real_2: 1.0188873904816649 (+0) + | > avg_loss_disc_real_3: 0.9964104091867487 (+0) + | > avg_loss_disc_real_4: 0.9932778782032906 (+0) + | > avg_loss_disc_real_5: 0.992817425981481 (+0) + | > avg_loss_0: 5.960158490120096 (+0) + | > avg_loss_gen: 5.958901770571445 (+0) + | > avg_loss_kl: 297.17751296023096 (+0) + | > avg_loss_feat: 0.4901856963938856 (+0) + | > avg_loss_mel: 101.03615975887216 (+0) + | > avg_loss_duration: 1.6993844889579934 (+0) + | > avg_loss_1: 406.36213781478557 (+0) + + + > EPOCH: 1/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 02:46:41)  + + --> TIME: 2024-06-25 02:46:50 -- STEP: 0/591 -- GLOBAL_STEP: 0 + | > loss_disc: 5.960390090942383 (5.960390090942383) + | > loss_disc_real_0: 0.9988713264465332 (0.9988713264465332) + | > loss_disc_real_1: 0.9594038724899292 (0.9594038724899292) + | > loss_disc_real_2: 1.019070029258728 (1.019070029258728) + | > loss_disc_real_3: 0.9963734149932861 (0.9963734149932861) + | > loss_disc_real_4: 0.9932559728622437 (0.9932559728622437) + | > loss_disc_real_5: 0.9928578734397888 (0.9928578734397888) + | > loss_0: 5.960390090942383 (5.960390090942383) + | > grad_norm_0: 0 (0) + | > loss_gen: 5.958893299102783 (5.958893299102783) + | > loss_kl: 301.5616455078125 (301.5616455078125) + | > loss_feat: 0.5683394074440002 (0.5683394074440002) + | > loss_mel: 101.52096557617188 (101.52096557617188) + | > loss_duration: 1.6349645853042603 (1.6349645853042603) + | > amp_scaler: 32768.0 (32768.0) + | > loss_1: 411.24481201171875 (411.24481201171875) + | > grad_norm_1: 0 (0) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 7.5481 (7.548060655593872) + | > loader_time: 0.7158 (0.7158288955688477) + + + --> TIME: 2024-06-25 02:47:21 -- STEP: 25/591 -- GLOBAL_STEP: 25 + | > loss_disc: 2.0910940170288086 (3.148499526977539) + | > loss_disc_real_0: 0.2676447927951813 (0.32984669744968415) + | > loss_disc_real_1: 0.19542530179023743 (0.32143471866846085) + | > loss_disc_real_2: 0.18063747882843018 (0.3202057939767837) + | > loss_disc_real_3: 0.15499712526798248 (0.31880480945110323) + | > loss_disc_real_4: 0.1543518602848053 (0.319261457324028) + | > loss_disc_real_5: 0.12657523155212402 (0.3246124452352524) + | > loss_0: 2.0910940170288086 (3.148499526977539) + | > grad_norm_0: tensor(8.7534, device='cuda:0') (tensor(3.4307, device='cuda:0')) + | > loss_gen: 2.970191717147827 (2.125186786651611) + | > loss_kl: 13.143774032592773 (113.03265865325928) + | > loss_feat: 4.265263080596924 (1.533210198879242) + | > loss_mel: 69.59270477294922 (82.81659759521484) + | > loss_duration: 1.4830498695373535 (1.5541542625427247) + | > amp_scaler: 512.0 (1679.36) + | > loss_1: 91.4549789428711 (201.0618118286133) + | > grad_norm_1: tensor(70.9043, device='cuda:0') (tensor(230.3256, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.3882 (1.2042945671081542) + | > loader_time: 0.0115 (0.009202566146850586) + + + --> TIME: 2024-06-25 02:47:48 -- STEP: 50/591 -- GLOBAL_STEP: 50 + | > loss_disc: 1.7559767961502075 (2.553511390686035) + | > loss_disc_real_0: 0.16500112414360046 (0.27037334591150275) + | > loss_disc_real_1: 0.21289001405239105 (0.2791349460184575) + | > loss_disc_real_2: 0.1949857473373413 (0.27713866770267487) + | > loss_disc_real_3: 0.17121168971061707 (0.272873567044735) + | > loss_disc_real_4: 0.1983216106891632 (0.26670107632875445) + | > loss_disc_real_5: 0.12043272703886032 (0.2600273804366589) + | > loss_0: 1.7559767961502075 (2.553511390686035) + | > grad_norm_0: tensor(7.3305, device='cuda:0') (tensor(5.7166, device='cuda:0')) + | > loss_gen: 2.789658784866333 (2.547462329864502) + | > loss_kl: 5.787533283233643 (60.78276840209965) + | > loss_feat: 6.333673000335693 (3.4911332499980925) + | > loss_mel: 52.15537643432617 (70.06875007629397) + | > loss_duration: 1.5610829591751099 (1.5437789344787598) + | > amp_scaler: 512.0 (1095.68) + | > loss_1: 68.6273193359375 (138.43389465332032) + | > grad_norm_1: tensor(103.4190, device='cuda:0') (tensor(163.2292, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.4441 (1.124687991142273) + | > loader_time: 0.0102 (0.009768705368041995) + + + --> TIME: 2024-06-25 02:48:17 -- STEP: 75/591 -- GLOBAL_STEP: 75 + | > loss_disc: 1.8132827281951904 (2.387939224243164) + | > loss_disc_real_0: 0.051617398858070374 (0.2284726656476656) + | > loss_disc_real_1: 0.18914169073104858 (0.26726765026648847) + | > loss_disc_real_2: 0.1814074069261551 (0.26457122762997953) + | > loss_disc_real_3: 0.1810702085494995 (0.2566214007139206) + | > loss_disc_real_4: 0.19404086470603943 (0.2492875212430955) + | > loss_disc_real_5: 0.15731939673423767 (0.23708153456449513) + | > loss_0: 1.8132827281951904 (2.387939224243164) + | > grad_norm_0: tensor(3.2166, device='cuda:0') (tensor(5.5608, device='cuda:0')) + | > loss_gen: 3.152963161468506 (2.6392875878016158) + | > loss_kl: 3.818661689758301 (42.00750649452212) + | > loss_feat: 5.69847297668457 (3.968814766407013) + | > loss_mel: 50.76606750488281 (63.8681992085775) + | > loss_duration: 1.5673513412475586 (1.5473791488011681) + | > amp_scaler: 512.0 (901.12) + | > loss_1: 65.0035171508789 (114.03118825276692) + | > grad_norm_1: tensor(67.6526, device='cuda:0') (tensor(148.9048, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.4362 (1.1277196820576982) + | > loader_time: 0.0119 (0.010070505142211916) + + + --> TIME: 2024-06-25 02:48:41 -- STEP: 100/591 -- GLOBAL_STEP: 100 + | > loss_disc: 2.744297504425049 (2.283448220491409) + | > loss_disc_real_0: 0.2655961513519287 (0.21215900547802444) + | > loss_disc_real_1: 0.3085353672504425 (0.2608470868319273) + | > loss_disc_real_2: 0.2996136248111725 (0.2572274208068848) + | > loss_disc_real_3: 0.26884207129478455 (0.24562844619154928) + | > loss_disc_real_4: 0.33159276843070984 (0.23973341345787055) + | > loss_disc_real_5: 0.3096526265144348 (0.2207258574664593) + | > loss_0: 2.744297504425049 (2.283448220491409) + | > grad_norm_0: tensor(5.1370, device='cuda:0') (tensor(6.3663, device='cuda:0')) + | > loss_gen: 2.218109607696533 (2.755532382726669) + | > loss_kl: 2.7653310298919678 (32.285649530887625) + | > loss_feat: 2.8206748962402344 (4.385498558878899) + | > loss_mel: 48.599327087402344 (60.373866844177265) + | > loss_duration: 1.5691769123077393 (1.5569695019721987) + | > amp_scaler: 512.0 (803.84) + | > loss_1: 57.972618103027344 (101.35751750946045) + | > grad_norm_1: tensor(179.2328, device='cuda:0') (tensor(138.2435, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.4507 (1.0868201565742495) + | > loader_time: 0.0108 (0.010384793281555178) + + + --> TIME: 2024-06-25 02:49:08 -- STEP: 125/591 -- GLOBAL_STEP: 125 + | > loss_disc: 1.9326167106628418 (2.2383647918701173) + | > loss_disc_real_0: 0.04754899814724922 (0.18862143394351003) + | > loss_disc_real_1: 0.2308765947818756 (0.2554072080254555) + | > loss_disc_real_2: 0.2609722912311554 (0.251156835436821) + | > loss_disc_real_3: 0.191193088889122 (0.23912784123420713) + | > loss_disc_real_4: 0.17159973084926605 (0.23252958703041082) + | > loss_disc_real_5: 0.19717726111412048 (0.21548020851612093) + | > loss_0: 1.9326167106628418 (2.2383647918701173) + | > grad_norm_0: tensor(7.6522, device='cuda:0') (tensor(6.3829, device='cuda:0')) + | > loss_gen: 2.8879451751708984 (2.739991793632507) + | > loss_kl: 2.492408037185669 (26.3288248023987) + | > loss_feat: 5.482741832733154 (4.449099915981293) + | > loss_mel: 48.706260681152344 (58.192774200439466) + | > loss_duration: 1.6381480693817139 (1.5615478038787844) + | > amp_scaler: 512.0 (745.472) + | > loss_1: 61.2075080871582 (93.27223913574218) + | > grad_norm_1: tensor(140.3918, device='cuda:0') (tensor(136.4483, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.5854 (1.0801438388824462) + | > loader_time: 0.0133 (0.010696510314941407) + + + --> TIME: 2024-06-25 02:49:35 -- STEP: 150/591 -- GLOBAL_STEP: 150 + | > loss_disc: 1.9596513509750366 (2.2433273196220402) + | > loss_disc_real_0: 0.24566811323165894 (0.19135821444292864) + | > loss_disc_real_1: 0.26525402069091797 (0.2522933225333691) + | > loss_disc_real_2: 0.25089219212532043 (0.24701626951495806) + | > loss_disc_real_3: 0.23287494480609894 (0.23668068985144297) + | > loss_disc_real_4: 0.15989772975444794 (0.22852113937338198) + | > loss_disc_real_5: 0.21482588350772858 (0.21593283707896868) + | > loss_0: 1.9596513509750366 (2.2433273196220402) + | > grad_norm_0: tensor(7.5850, device='cuda:0') (tensor(7.4574, device='cuda:0')) + | > loss_gen: 3.117553949356079 (2.733575881322225) + | > loss_kl: 1.9256170988082886 (22.303574731349958) + | > loss_feat: 5.1996917724609375 (4.44439701517423) + | > loss_mel: 47.98912811279297 (56.59050209045412) + | > loss_duration: 1.515437126159668 (1.5628114374478659) + | > amp_scaler: 512.0 (706.56) + | > loss_1: 59.74742889404297 (87.63486172993977) + | > grad_norm_1: tensor(109.4586, device='cuda:0') (tensor(131.2385, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.8894 (1.0749706808725996) + | > loader_time: 0.0136 (0.010915559132893887) + + + --> TIME: 2024-06-25 02:50:03 -- STEP: 175/591 -- GLOBAL_STEP: 175 + | > loss_disc: 2.4710419178009033 (2.2696996981757036) + | > loss_disc_real_0: 0.20568904280662537 (0.18926880476730215) + | > loss_disc_real_1: 0.18379129469394684 (0.25115966111421584) + | > loss_disc_real_2: 0.15710657835006714 (0.24734992465802602) + | > loss_disc_real_3: 0.2766958177089691 (0.23713116466999054) + | > loss_disc_real_4: 0.18161240220069885 (0.2256317280871528) + | > loss_disc_real_5: 0.13927589356899261 (0.21562739121062416) + | > loss_0: 2.4710419178009033 (2.2696996981757036) + | > grad_norm_0: tensor(10.2437, device='cuda:0') (tensor(7.7786, device='cuda:0')) + | > loss_gen: 2.248929977416992 (2.700402634484427) + | > loss_kl: 2.677394390106201 (19.444301476478582) + | > loss_feat: 4.411509990692139 (4.356112738677434) + | > loss_mel: 44.69345474243164 (55.21425524030415) + | > loss_duration: 1.5593101978302002 (1.5623646129880633) + | > amp_scaler: 512.0 (678.7657142857141) + | > loss_1: 55.590599060058594 (83.27743730817518) + | > grad_norm_1: tensor(56.8657, device='cuda:0') (tensor(125.3884, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.5533 (1.0831992040361675) + | > loader_time: 0.0125 (0.011145538602556508) + + + --> TIME: 2024-06-25 02:50:30 -- STEP: 200/591 -- GLOBAL_STEP: 200 + | > loss_disc: 2.130577564239502 (2.2670197570323953) + | > loss_disc_real_0: 0.14822402596473694 (0.19317882625386124) + | > loss_disc_real_1: 0.16686896979808807 (0.24883459407836192) + | > loss_disc_real_2: 0.18132543563842773 (0.24492375154048204) + | > loss_disc_real_3: 0.2322724610567093 (0.23584414221346378) + | > loss_disc_real_4: 0.16171853244304657 (0.21720871876925232) + | > loss_disc_real_5: 0.2084008902311325 (0.21199381370097398) + | > loss_0: 2.130577564239502 (2.2670197570323953) + | > grad_norm_0: tensor(4.9547, device='cuda:0') (tensor(7.9409, device='cuda:0')) + | > loss_gen: 2.2414584159851074 (2.6907746851444245) + | > loss_kl: 2.280242681503296 (17.304270732998848) + | > loss_feat: 4.238977909088135 (4.348153328597548) + | > loss_mel: 42.40590286254883 (53.76789192199709) + | > loss_duration: 1.5663893222808838 (1.562931984066963) + | > amp_scaler: 512.0 (657.92) + | > loss_1: 52.732975006103516 (79.6740232086181) + | > grad_norm_1: tensor(148.0445, device='cuda:0') (tensor(123.7293, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.5512 (1.0795740485191343) + | > loader_time: 0.013 (0.01134968042373658) + + + --> TIME: 2024-06-25 02:50:57 -- STEP: 225/591 -- GLOBAL_STEP: 225 + | > loss_disc: 1.858371376991272 (2.245480956501433) + | > loss_disc_real_0: 0.210473895072937 (0.1924155818588204) + | > loss_disc_real_1: 0.23937150835990906 (0.2464144626922077) + | > loss_disc_real_2: 0.21788936853408813 (0.24167773673931758) + | > loss_disc_real_3: 0.24953298270702362 (0.23229405012395646) + | > loss_disc_real_4: 0.18375487625598907 (0.21621308929390382) + | > loss_disc_real_5: 0.10077165812253952 (0.2055825106302897) + | > loss_0: 1.858371376991272 (2.245480956501433) + | > grad_norm_0: tensor(10.4854, device='cuda:0') (tensor(8.3054, device='cuda:0')) + | > loss_gen: 2.765998363494873 (2.6968445449405247) + | > loss_kl: 2.2195050716400146 (15.629363320138717) + | > loss_feat: 5.634859085083008 (4.387283878061509) + | > loss_mel: 41.89101028442383 (52.49534606933595) + | > loss_duration: 1.5571956634521484 (1.5635592444737751) + | > amp_scaler: 512.0 (641.7066666666667) + | > loss_1: 54.068572998046875 (76.77239759657111) + | > grad_norm_1: tensor(109.7598, device='cuda:0') (tensor(122.9093, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.5658 (1.0770478841993536) + | > loader_time: 0.0161 (0.011682197782728413) + + + --> TIME: 2024-06-25 02:51:27 -- STEP: 250/591 -- GLOBAL_STEP: 250 + | > loss_disc: 2.7946109771728516 (2.2451305027008073) + | > loss_disc_real_0: 0.2619871497154236 (0.19275413002073769) + | > loss_disc_real_1: 0.3464096784591675 (0.24680943384766574) + | > loss_disc_real_2: 0.39610040187835693 (0.24229598054289816) + | > loss_disc_real_3: 0.3747226595878601 (0.2321135911345482) + | > loss_disc_real_4: 0.24898217618465424 (0.21144237861037257) + | > loss_disc_real_5: 0.29169705510139465 (0.2024051384627819) + | > loss_0: 2.7946109771728516 (2.2451305027008073) + | > grad_norm_0: tensor(12.9033, device='cuda:0') (tensor(8.6288, device='cuda:0')) + | > loss_gen: 2.1720728874206543 (2.705407485961914) + | > loss_kl: 2.018038272857666 (14.280670518398283) + | > loss_feat: 3.3507914543151855 (4.400009209394458) + | > loss_mel: 42.048583984375 (51.4558132171631) + | > loss_duration: 1.5507563352584839 (1.5649363913536072) + | > amp_scaler: 512.0 (628.736) + | > loss_1: 51.14023971557617 (74.40683731079096) + | > grad_norm_1: tensor(106.7514, device='cuda:0') (tensor(121.3889, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.5637 (1.0861211795806878) + | > loader_time: 0.0169 (0.012016016960144048) + + + --> TIME: 2024-06-25 02:51:55 -- STEP: 275/591 -- GLOBAL_STEP: 275 + | > loss_disc: 2.6605448722839355 (2.239440993829209) + | > loss_disc_real_0: 0.25627225637435913 (0.1936649989404462) + | > loss_disc_real_1: 0.2639485001564026 (0.24527320284735069) + | > loss_disc_real_2: 0.2912950813770294 (0.2408288031003692) + | > loss_disc_real_3: 0.275331050157547 (0.23078523684631694) + | > loss_disc_real_4: 0.21856188774108887 (0.20695950036699118) + | > loss_disc_real_5: 0.43228787183761597 (0.1996051280877807) + | > loss_0: 2.6605448722839355 (2.239440993829209) + | > grad_norm_0: tensor(28.0899, device='cuda:0') (tensor(8.6346, device='cuda:0')) + | > loss_gen: 2.0437817573547363 (2.7099914438074286) + | > loss_kl: 2.175964117050171 (13.17505602273074) + | > loss_feat: 4.330289363861084 (4.427096510583706) + | > loss_mel: 40.653690338134766 (50.50487709738993) + | > loss_duration: 1.581811785697937 (1.5662438895485618) + | > amp_scaler: 512.0 (618.1236363636364) + | > loss_1: 50.7855339050293 (72.38326536698769) + | > grad_norm_1: tensor(128.5920, device='cuda:0') (tensor(121.6794, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.5868 (1.0872979857704852) + | > loader_time: 0.0165 (0.012275840585882016) + + + --> TIME: 2024-06-25 02:52:22 -- STEP: 300/591 -- GLOBAL_STEP: 300 + | > loss_disc: 2.195028305053711 (2.2353320523103104) + | > loss_disc_real_0: 0.2810078263282776 (0.19374884050339466) + | > loss_disc_real_1: 0.20076584815979004 (0.24393492651482418) + | > loss_disc_real_2: 0.2970225512981415 (0.23952517800033093) + | > loss_disc_real_3: 0.18473777174949646 (0.2291348498314619) + | > loss_disc_real_4: 0.19530843198299408 (0.2036331449697415) + | > loss_disc_real_5: 0.20155483484268188 (0.19911456753810247) + | > loss_0: 2.195028305053711 (2.2353320523103104) + | > grad_norm_0: tensor(7.2122, device='cuda:0') (tensor(8.9998, device='cuda:0')) + | > loss_gen: 3.27628231048584 (2.7116797089576723) + | > loss_kl: 2.1009674072265625 (12.249232779343924) + | > loss_feat: 4.464666366577148 (4.4368590114514035) + | > loss_mel: 39.40262222290039 (49.640009371439625) + | > loss_duration: 1.6194850206375122 (1.5680065699418386) + | > amp_scaler: 512.0 (609.28) + | > loss_1: 50.86402130126953 (70.60578777313228) + | > grad_norm_1: tensor(218.2098, device='cuda:0') (tensor(122.5216, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.6463 (1.0870514535903928) + | > loader_time: 0.0169 (0.01255642414093018) + + + --> TIME: 2024-06-25 02:52:53 -- STEP: 325/591 -- GLOBAL_STEP: 325 + | > loss_disc: 2.2461225986480713 (2.2390139715488164) + | > loss_disc_real_0: 0.20448115468025208 (0.19508181637296315) + | > loss_disc_real_1: 0.2574501633644104 (0.24327097807939232) + | > loss_disc_real_2: 0.2928515076637268 (0.23891190888789984) + | > loss_disc_real_3: 0.26293763518333435 (0.22875392517218224) + | > loss_disc_real_4: 0.14724485576152802 (0.2028517950727389) + | > loss_disc_real_5: 0.2938026189804077 (0.19979931803850032) + | > loss_0: 2.2461225986480713 (2.2390139715488164) + | > grad_norm_0: tensor(14.8571, device='cuda:0') (tensor(9.1011, device='cuda:0')) + | > loss_gen: 2.8405208587646484 (2.700460432492769) + | > loss_kl: 1.9977246522903442 (11.464942076756406) + | > loss_feat: 4.333864212036133 (4.415388804032252) + | > loss_mel: 39.4761962890625 (48.8903073002742) + | > loss_duration: 1.6367676258087158 (1.5699801734777605) + | > amp_scaler: 512.0 (601.796923076923) + | > loss_1: 50.28507614135742 (69.04107911329997) + | > grad_norm_1: tensor(161.2276, device='cuda:0') (tensor(123.8316, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.6282 (1.0959365206498364) + | > loader_time: 0.0148 (0.012827321566068214) + + + --> TIME: 2024-06-25 02:53:23 -- STEP: 350/591 -- GLOBAL_STEP: 350 + | > loss_disc: 2.280874013900757 (2.246649934564321) + | > loss_disc_real_0: 0.19355951249599457 (0.194841209979994) + | > loss_disc_real_1: 0.2443198561668396 (0.2435576978538717) + | > loss_disc_real_2: 0.29646509885787964 (0.23875598232660974) + | > loss_disc_real_3: 0.31024184823036194 (0.22893355840018817) + | > loss_disc_real_4: 0.1790309101343155 (0.2022791625346456) + | > loss_disc_real_5: 0.23367957770824432 (0.1995721132414682) + | > loss_0: 2.280874013900757 (2.246649934564321) + | > grad_norm_0: tensor(6.9226, device='cuda:0') (tensor(8.9786, device='cuda:0')) + | > loss_gen: 2.393007278442383 (2.6817427560261318) + | > loss_kl: 1.8990821838378906 (10.786788171359479) + | > loss_feat: 3.8029251098632812 (4.3704677259921985) + | > loss_mel: 38.37644577026367 (48.235539016723656) + | > loss_duration: 1.6098847389221191 (1.5723776323454728) + | > amp_scaler: 512.0 (595.382857142857) + | > loss_1: 48.08134460449219 (67.64691557747967) + | > grad_norm_1: tensor(162.2840, device='cuda:0') (tensor(124.0512, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.7128 (1.1024979713984893) + | > loader_time: 0.0175 (0.013133217947823666) + + + --> TIME: 2024-06-25 02:53:53 -- STEP: 375/591 -- GLOBAL_STEP: 375 + | > loss_disc: 2.217771291732788 (2.255161409060163) + | > loss_disc_real_0: 0.16166338324546814 (0.19473554284373923) + | > loss_disc_real_1: 0.29277873039245605 (0.2432632262508074) + | > loss_disc_real_2: 0.20635031163692474 (0.23783540429671604) + | > loss_disc_real_3: 0.2589229643344879 (0.2291452047030131) + | > loss_disc_real_4: 0.2017408311367035 (0.20329536004861193) + | > loss_disc_real_5: 0.18818385899066925 (0.19949817685286209) + | > loss_0: 2.217771291732788 (2.255161409060163) + | > grad_norm_0: tensor(3.9518, device='cuda:0') (tensor(9.1792, device='cuda:0')) + | > loss_gen: 2.389399290084839 (2.6651020269393926) + | > loss_kl: 1.9143037796020508 (10.199677828152986) + | > loss_feat: 3.7664144039154053 (4.326126055876411) + | > loss_mel: 38.200523376464844 (47.61930771891278) + | > loss_duration: 1.6203036308288574 (1.574773627599081) + | > amp_scaler: 512.0 (589.8239999999998) + | > loss_1: 47.89094543457031 (66.38498751831047) + | > grad_norm_1: tensor(112.7110, device='cuda:0') (tensor(123.9395, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.7622 (1.106681866963703) + | > loader_time: 0.0188 (0.013401500701904302) + + + --> TIME: 2024-06-25 02:54:23 -- STEP: 400/591 -- GLOBAL_STEP: 400 + | > loss_disc: 2.4524917602539062 (2.261815650165083) + | > loss_disc_real_0: 0.18945923447608948 (0.19507726944051687) + | > loss_disc_real_1: 0.2382359802722931 (0.2427263531647622) + | > loss_disc_real_2: 0.1964852213859558 (0.23749379167333245) + | > loss_disc_real_3: 0.20716486871242523 (0.22885837227106096) + | > loss_disc_real_4: 0.2013254016637802 (0.20380836613476272) + | > loss_disc_real_5: 0.18359051644802094 (0.199186513312161) + | > loss_0: 2.4524917602539062 (2.261815650165083) + | > grad_norm_0: tensor(5.1033, device='cuda:0') (tensor(9.1639, device='cuda:0')) + | > loss_gen: 2.205078363418579 (2.649178820848465) + | > loss_kl: 1.9832793474197388 (9.685797128677372) + | > loss_feat: 3.536858558654785 (4.291889858692881) + | > loss_mel: 38.5732536315918 (47.07618661880495) + | > loss_duration: 1.6231188774108887 (1.576907361745835) + | > amp_scaler: 512.0 (584.96) + | > loss_1: 47.92158889770508 (65.2799600028991) + | > grad_norm_1: tensor(163.7797, device='cuda:0') (tensor(125.7729, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.7207 (1.1128627306222902) + | > loader_time: 0.0191 (0.013669034838676456) + + + --> TIME: 2024-06-25 02:54:56 -- STEP: 425/591 -- GLOBAL_STEP: 425 + | > loss_disc: 2.529592990875244 (2.2773118846556746) + | > loss_disc_real_0: 0.16853202879428864 (0.19575779575635407) + | > loss_disc_real_1: 0.20075687766075134 (0.24257263890084094) + | > loss_disc_real_2: 0.2257196605205536 (0.23718203479752822) + | > loss_disc_real_3: 0.20019078254699707 (0.22953943382291234) + | > loss_disc_real_4: 0.21230265498161316 (0.20434915463714035) + | > loss_disc_real_5: 0.17318101227283478 (0.1995035047390882) + | > loss_0: 2.529592990875244 (2.2773118846556746) + | > grad_norm_0: tensor(6.2091, device='cuda:0') (tensor(9.0540, device='cuda:0')) + | > loss_gen: 2.5120248794555664 (2.621683241900276) + | > loss_kl: 1.8961213827133179 (9.230442310501553) + | > loss_feat: 3.417217969894409 (4.216425327833959) + | > loss_mel: 38.628807067871094 (46.55471869973577) + | > loss_duration: 1.626840353012085 (1.5787318950540885) + | > amp_scaler: 512.0 (580.6682352941178) + | > loss_1: 48.08100891113281 (64.20200168385219) + | > grad_norm_1: tensor(201.1618, device='cuda:0') (tensor(126.5375, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.7923 (1.1229870347415687) + | > loader_time: 0.0198 (0.013961151347440836) + + + --> TIME: 2024-06-25 02:55:29 -- STEP: 450/591 -- GLOBAL_STEP: 450 + | > loss_disc: 2.575888156890869 (2.2921158395873196) + | > loss_disc_real_0: 0.29941046237945557 (0.19767942353255222) + | > loss_disc_real_1: 0.2536071538925171 (0.24271572723984716) + | > loss_disc_real_2: 0.2645721137523651 (0.23715321489506297) + | > loss_disc_real_3: 0.24713601171970367 (0.22979586862855486) + | > loss_disc_real_4: 0.22091048955917358 (0.20537204371558293) + | > loss_disc_real_5: 0.2628408968448639 (0.20083821391065917) + | > loss_0: 2.575888156890869 (2.2921158395873196) + | > grad_norm_0: tensor(7.3403, device='cuda:0') (tensor(9.0769, device='cuda:0')) + | > loss_gen: 2.068901300430298 (2.6002112640274895) + | > loss_kl: 1.9625753164291382 (8.826107388602367) + | > loss_feat: 2.696404218673706 (4.157279190884691) + | > loss_mel: 38.23493194580078 (46.08168718126087) + | > loss_duration: 1.627366065979004 (1.5809422196282286) + | > amp_scaler: 512.0 (576.8533333333337) + | > loss_1: 46.590179443359375 (63.24622745937766) + | > grad_norm_1: tensor(133.7316, device='cuda:0') (tensor(127.8884, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.7639 (1.1324566067589645) + | > loader_time: 0.0192 (0.014213590092129181) + + + --> TIME: 2024-06-25 02:56:01 -- STEP: 475/591 -- GLOBAL_STEP: 475 + | > loss_disc: 2.4996604919433594 (2.302037794464515) + | > loss_disc_real_0: 0.3024630546569824 (0.19842129711257786) + | > loss_disc_real_1: 0.2836853861808777 (0.2423870393633842) + | > loss_disc_real_2: 0.2848374843597412 (0.23689554140756006) + | > loss_disc_real_3: 0.2339654266834259 (0.22968656125821565) + | > loss_disc_real_4: 0.2893248200416565 (0.20599184111544958) + | > loss_disc_real_5: 0.31700971722602844 (0.2018318838037943) + | > loss_0: 2.4996604919433594 (2.302037794464515) + | > grad_norm_0: tensor(12.3029, device='cuda:0') (tensor(8.9971, device='cuda:0')) + | > loss_gen: 2.2168033123016357 (2.579278137558385) + | > loss_kl: 1.8693697452545166 (8.46141731764141) + | > loss_feat: 3.2963321208953857 (4.100465518926314) + | > loss_mel: 37.093196868896484 (45.63076159828589) + | > loss_duration: 1.6677663326263428 (1.5833507444984039) + | > amp_scaler: 512.0 (573.4400000000003) + | > loss_1: 46.143470764160156 (62.355273517809366) + | > grad_norm_1: tensor(176.5518, device='cuda:0') (tensor(129.8488, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.7843 (1.1390799813521526) + | > loader_time: 0.0167 (0.014460766942877522) + + + --> TIME: 2024-06-25 02:56:38 -- STEP: 500/591 -- GLOBAL_STEP: 500 + | > loss_disc: 2.5672965049743652 (2.313525579690935) + | > loss_disc_real_0: 0.1597181111574173 (0.198639788992703) + | > loss_disc_real_1: 0.2614854574203491 (0.24229773025214668) + | > loss_disc_real_2: 0.2359255850315094 (0.23667560844123364) + | > loss_disc_real_3: 0.27175894379615784 (0.2299168939590454) + | > loss_disc_real_4: 0.252579927444458 (0.20752673406898972) + | > loss_disc_real_5: 0.2733864486217499 (0.20232281275093558) + | > loss_0: 2.5672965049743652 (2.313525579690935) + | > grad_norm_0: tensor(5.0227, device='cuda:0') (tensor(9.0157, device='cuda:0')) + | > loss_gen: 2.143218994140625 (2.5592253749370575) + | > loss_kl: 2.0305657386779785 (8.132641484022143) + | > loss_feat: 3.1291983127593994 (4.045630499243728) + | > loss_mel: 36.402435302734375 (45.20578916931154) + | > loss_duration: 1.631216287612915 (1.5854688429832462) + | > amp_scaler: 512.0 (570.3680000000003) + | > loss_1: 45.33663558959961 (61.52875560760493) + | > grad_norm_1: tensor(133.5947, device='cuda:0') (tensor(131.2308, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.7655 (1.1537407755851736) + | > loader_time: 0.0255 (0.014725955009460452) + + + --> TIME: 2024-06-25 02:57:13 -- STEP: 525/591 -- GLOBAL_STEP: 525 + | > loss_disc: 2.7016396522521973 (2.329597328958059) + | > loss_disc_real_0: 0.17775984108448029 (0.19927417911943937) + | > loss_disc_real_1: 0.18430085480213165 (0.24217877105588) + | > loss_disc_real_2: 0.279377818107605 (0.2375999354606583) + | > loss_disc_real_3: 0.14961573481559753 (0.2301146709067481) + | > loss_disc_real_4: 0.265330046415329 (0.20868470741169792) + | > loss_disc_real_5: 0.19192764163017273 (0.20369863434916455) + | > loss_0: 2.7016396522521973 (2.329597328958059) + | > grad_norm_0: tensor(7.5528, device='cuda:0') (tensor(8.9216, device='cuda:0')) + | > loss_gen: 1.9312186241149902 (2.5370997519720166) + | > loss_kl: 1.8332918882369995 (7.833933628173101) + | > loss_feat: 2.5119431018829346 (3.983819738456171) + | > loss_mel: 38.582759857177734 (44.82857050577801) + | > loss_duration: 1.6710116863250732 (1.5884231742223107) + | > amp_scaler: 512.0 (567.5885714285716) + | > loss_1: 46.53022384643555 (60.77184700375507) + | > grad_norm_1: tensor(127.4730, device='cuda:0') (tensor(132.1571, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.6986 (1.1658969293321875) + | > loader_time: 0.0218 (0.015029496692475821) + + + --> TIME: 2024-06-25 02:57:50 -- STEP: 550/591 -- GLOBAL_STEP: 550 + | > loss_disc: 2.5787158012390137 (2.3392744560675194) + | > loss_disc_real_0: 0.13969947397708893 (0.19961756434630265) + | > loss_disc_real_1: 0.28519952297210693 (0.24226658697832715) + | > loss_disc_real_2: 0.2877654433250427 (0.23711186662316322) + | > loss_disc_real_3: 0.2333802431821823 (0.23003694699569185) + | > loss_disc_real_4: 0.29182302951812744 (0.2093222596428611) + | > loss_disc_real_5: 0.363166868686676 (0.20458228628743783) + | > loss_0: 2.5787158012390137 (2.3392744560675194) + | > grad_norm_0: tensor(10.9250, device='cuda:0') (tensor(8.8356, device='cuda:0')) + | > loss_gen: 1.880366563796997 (2.5177699340473523) + | > loss_kl: 2.0184948444366455 (7.5628336271372705) + | > loss_feat: 2.7305195331573486 (3.9310155607353456) + | > loss_mel: 36.50502014160156 (44.46896929654209) + | > loss_duration: 1.636805772781372 (1.5910571410439236) + | > amp_scaler: 512.0 (565.0618181818184) + | > loss_1: 44.771209716796875 (60.071645764437534) + | > grad_norm_1: tensor(208.6140, device='cuda:0') (tensor(132.4653, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 1.9454 (1.17728813604875) + | > loader_time: 0.02 (0.01535187201066451) + + + --> TIME: 2024-06-25 02:58:32 -- STEP: 575/591 -- GLOBAL_STEP: 575 + | > loss_disc: 2.437666416168213 (2.349270168594693) + | > loss_disc_real_0: 0.24755901098251343 (0.20030659077608062) + | > loss_disc_real_1: 0.24878831207752228 (0.24220534779455352) + | > loss_disc_real_2: 0.28086885809898376 (0.23712342781865078) + | > loss_disc_real_3: 0.22432328760623932 (0.23068710333627204) + | > loss_disc_real_4: 0.3121674358844757 (0.21039451373660048) + | > loss_disc_real_5: 0.2922741770744324 (0.20507668238619106) + | > loss_0: 2.437666416168213 (2.349270168594693) + | > grad_norm_0: tensor(10.2356, device='cuda:0') (tensor(8.8838, device='cuda:0')) + | > loss_gen: 2.0914134979248047 (2.5040465257478806) + | > loss_kl: 1.8388015031814575 (7.313870019290758) + | > loss_feat: 2.9752092361450195 (3.888074358960847) + | > loss_mel: 37.642799377441406 (44.12033429353135) + | > loss_duration: 1.6866066455841064 (1.5937555800313539) + | > amp_scaler: 512.0 (562.7547826086959) + | > loss_1: 46.234832763671875 (59.42008097773008) + | > grad_norm_1: tensor(186.8460, device='cuda:0') (tensor(133.6650, device='cuda:0')) + | > current_lr_0: 0.0002 + | > current_lr_1: 0.0002 + | > step_time: 2.4212 (1.19884640900985) + | > loader_time: 0.0249 (0.01571889960247538) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038619092170228348 (-0.0001577012082363688) + | > avg_loss_disc: 2.5846205924419645 (-3.375537897678132) + | > avg_loss_disc_real_0: 0.21216580049788697 (-0.7866680916953595) + | > avg_loss_disc_real_1: 0.21226297445753786 (-0.7471109917189213) + | > avg_loss_disc_real_2: 0.21703530689503284 (-0.8018520835866321) + | > avg_loss_disc_real_3: 0.18482056291813545 (-0.8115898462686133) + | > avg_loss_disc_real_4: 0.17069124493827212 (-0.8225866332650185) + | > avg_loss_disc_real_5: 0.14305436238646507 (-0.849763063595016) + | > avg_loss_0: 2.5846205924419645 (-3.375537897678132) + | > avg_loss_gen: 1.7628888855589198 (-4.196012885012525) + | > avg_loss_kl: 2.1385930624413994 (-295.0389198977896) + | > avg_loss_feat: 2.801606127556334 (+2.3114204311624484) + | > avg_loss_mel: 36.90158754713992 (-64.13457221173223) + | > avg_loss_duration: 1.5801447401655482 (-0.11923974879244525) + | > avg_loss_1: 45.18482062157164 (-361.1773171932139) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_591.pth + + > EPOCH: 2/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 02:59:11)  + + --> TIME: 2024-06-25 02:59:20 -- STEP: 9/591 -- GLOBAL_STEP: 600 + | > loss_disc: 2.2933151721954346 (2.5362831486596003) + | > loss_disc_real_0: 0.15234363079071045 (0.19427221185631222) + | > loss_disc_real_1: 0.1657569855451584 (0.2548895561032825) + | > loss_disc_real_2: 0.1598275601863861 (0.24676765004793802) + | > loss_disc_real_3: 0.2041003555059433 (0.251347874601682) + | > loss_disc_real_4: 0.16578912734985352 (0.22596119509802925) + | > loss_disc_real_5: 0.12770259380340576 (0.24728369216124216) + | > loss_0: 2.2933151721954346 (2.5362831486596003) + | > grad_norm_0: tensor(7.7953, device='cuda:0') (tensor(7.9313, device='cuda:0')) + | > loss_gen: 1.9354915618896484 (2.1707837846544056) + | > loss_kl: 1.9644966125488281 (2.024752285745409) + | > loss_feat: 3.588224411010742 (3.1554464764065213) + | > loss_mel: 37.923587799072266 (38.143814934624565) + | > loss_duration: 1.4654202461242676 (1.512809567981296) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 46.87722396850586 (47.00760862562392) + | > grad_norm_1: tensor(64.6788, device='cuda:0') (tensor(186.0913, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.7895 (0.7697386211819119) + | > loader_time: 0.0096 (0.008731153276231553) + + + --> TIME: 2024-06-25 02:59:40 -- STEP: 34/591 -- GLOBAL_STEP: 625 + | > loss_disc: 2.5717601776123047 (2.5598284146365) + | > loss_disc_real_0: 0.11164961010217667 (0.20095721084405394) + | > loss_disc_real_1: 0.20598949491977692 (0.24934977117706747) + | > loss_disc_real_2: 0.15721796452999115 (0.24133789144894657) + | > loss_disc_real_3: 0.14670315384864807 (0.23548942979644327) + | > loss_disc_real_4: 0.15339361131191254 (0.21700205829213648) + | > loss_disc_real_5: 0.14043264091014862 (0.23021160548224168) + | > loss_0: 2.5717601776123047 (2.5598284146365) + | > grad_norm_0: tensor(14.9099, device='cuda:0') (tensor(8.2317, device='cuda:0')) + | > loss_gen: 2.176279067993164 (2.161213962470784) + | > loss_kl: 1.9415475130081177 (2.00828971932916) + | > loss_feat: 3.404390335083008 (3.014102858655593) + | > loss_mel: 38.472991943359375 (37.54058097390566) + | > loss_duration: 1.5825858116149902 (1.5311034532154308) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 47.57779312133789 (46.255291658289295) + | > grad_norm_1: tensor(62.3785, device='cuda:0') (tensor(141.4599, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.8013 (0.7840376391130335) + | > loader_time: 0.0114 (0.00939299779779771) + + + --> TIME: 2024-06-25 03:00:01 -- STEP: 59/591 -- GLOBAL_STEP: 650 + | > loss_disc: 2.4126622676849365 (2.562050346600807) + | > loss_disc_real_0: 0.2249433696269989 (0.20192646235227585) + | > loss_disc_real_1: 0.20843319594860077 (0.24443467592788956) + | > loss_disc_real_2: 0.2180829644203186 (0.2396266240689714) + | > loss_disc_real_3: 0.20198510587215424 (0.23642303302126416) + | > loss_disc_real_4: 0.16779401898384094 (0.22589450392682672) + | > loss_disc_real_5: 0.15412098169326782 (0.2210017377037113) + | > loss_0: 2.4126622676849365 (2.562050346600807) + | > grad_norm_0: tensor(8.0899, device='cuda:0') (tensor(7.6313, device='cuda:0')) + | > loss_gen: 2.3844738006591797 (2.1500472577951726) + | > loss_kl: 1.895961880683899 (1.9716983488050552) + | > loss_feat: 3.1134846210479736 (2.982763852103282) + | > loss_mel: 35.45665740966797 (37.054623619984774) + | > loss_duration: 1.5486207008361816 (1.5505935842708005) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 44.399200439453125 (45.70972720647263) + | > grad_norm_1: tensor(182.4698, device='cuda:0') (tensor(142.2022, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.8105 (0.7953289807853052) + | > loader_time: 0.0102 (0.009888838913481116) + + + --> TIME: 2024-06-25 03:00:22 -- STEP: 84/591 -- GLOBAL_STEP: 675 + | > loss_disc: 2.4593162536621094 (2.565348452045804) + | > loss_disc_real_0: 0.18856856226921082 (0.2067190736000027) + | > loss_disc_real_1: 0.29233646392822266 (0.2435471692255565) + | > loss_disc_real_2: 0.27711668610572815 (0.23835414594837598) + | > loss_disc_real_3: 0.24738946557044983 (0.23565352690361796) + | > loss_disc_real_4: 0.2254418283700943 (0.22584582838628972) + | > loss_disc_real_5: 0.31781357526779175 (0.22232485083597048) + | > loss_0: 2.4593162536621094 (2.565348452045804) + | > grad_norm_0: tensor(5.7145, device='cuda:0') (tensor(7.9991, device='cuda:0')) + | > loss_gen: 2.1726338863372803 (2.1481258925937463) + | > loss_kl: 2.235025405883789 (1.976331966263907) + | > loss_feat: 3.1637001037597656 (2.975023797580174) + | > loss_mel: 35.358848571777344 (36.814374060857865) + | > loss_duration: 1.5815999507904053 (1.5632120285715376) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 44.511810302734375 (45.47706826527914) + | > grad_norm_1: tensor(106.3520, device='cuda:0') (tensor(159.8124, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.8294 (0.803602127801804) + | > loader_time: 0.0125 (0.010290614196232385) + + + --> TIME: 2024-06-25 03:00:43 -- STEP: 109/591 -- GLOBAL_STEP: 700 + | > loss_disc: 2.400447368621826 (2.556816328556166) + | > loss_disc_real_0: 0.20845505595207214 (0.20885979042414132) + | > loss_disc_real_1: 0.1820479780435562 (0.24069224284329546) + | > loss_disc_real_2: 0.2088475376367569 (0.2353100238043234) + | > loss_disc_real_3: 0.26238521933555603 (0.23345669721244672) + | > loss_disc_real_4: 0.3124105632305145 (0.22571686409208752) + | > loss_disc_real_5: 0.1644611656665802 (0.21923904517375) + | > loss_0: 2.400447368621826 (2.556816328556166) + | > grad_norm_0: tensor(6.7508, device='cuda:0') (tensor(8.0015, device='cuda:0')) + | > loss_gen: 2.334320068359375 (2.1544420358237857) + | > loss_kl: 2.031930685043335 (1.9790880723830755) + | > loss_feat: 3.5548722743988037 (3.0122838545283046) + | > loss_mel: 36.82669448852539 (36.67431528633886) + | > loss_duration: 1.600874900817871 (1.5719769646268371) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 46.34869384765625 (45.39210667741408) + | > grad_norm_1: tensor(152.8845, device='cuda:0') (tensor(164.7839, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.8307 (0.810967456310167) + | > loader_time: 0.011 (0.010573857421175056) + + + --> TIME: 2024-06-25 03:01:04 -- STEP: 134/591 -- GLOBAL_STEP: 725 + | > loss_disc: 2.553067445755005 (2.5479378949350386) + | > loss_disc_real_0: 0.15215766429901123 (0.20866800916951095) + | > loss_disc_real_1: 0.15343545377254486 (0.23974059377588444) + | > loss_disc_real_2: 0.16186007857322693 (0.23429899384726338) + | > loss_disc_real_3: 0.1927947998046875 (0.2337674975617608) + | > loss_disc_real_4: 0.16398997604846954 (0.2248304306682366) + | > loss_disc_real_5: 0.17045918107032776 (0.21940163128189186) + | > loss_0: 2.553067445755005 (2.5479378949350386) + | > grad_norm_0: tensor(9.7649, device='cuda:0') (tensor(7.7838, device='cuda:0')) + | > loss_gen: 1.9254827499389648 (2.162069187235475) + | > loss_kl: 1.99652898311615 (1.9816377207414428) + | > loss_feat: 3.2240185737609863 (3.0518412732366307) + | > loss_mel: 35.608917236328125 (36.598569015958404) + | > loss_duration: 1.5670076608657837 (1.5746898037284167) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 44.321956634521484 (45.36880749375073) + | > grad_norm_1: tensor(143.5738, device='cuda:0') (tensor(169.7347, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.8417 (0.8167283072400446) + | > loader_time: 0.0124 (0.01082240467640891) + + + --> TIME: 2024-06-25 03:01:26 -- STEP: 159/591 -- GLOBAL_STEP: 750 + | > loss_disc: 2.4380760192871094 (2.541520064731814) + | > loss_disc_real_0: 0.19005008041858673 (0.20835103146122685) + | > loss_disc_real_1: 0.2220059633255005 (0.2392034241988224) + | > loss_disc_real_2: 0.17537540197372437 (0.23218907003507674) + | > loss_disc_real_3: 0.20747679471969604 (0.23296711870334433) + | > loss_disc_real_4: 0.24015696346759796 (0.22382937732545086) + | > loss_disc_real_5: 0.17743505537509918 (0.21782110364369625) + | > loss_0: 2.4380760192871094 (2.541520064731814) + | > grad_norm_0: tensor(5.6520, device='cuda:0') (tensor(7.6014, device='cuda:0')) + | > loss_gen: 2.074688673019409 (2.157615415705075) + | > loss_kl: 1.9781715869903564 (1.9848343703731801) + | > loss_feat: 3.2036445140838623 (3.046654012967956) + | > loss_mel: 35.84850311279297 (36.422242530486855) + | > loss_duration: 1.564894199371338 (1.5741809006756957) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 44.66990280151367 (45.18552775353007) + | > grad_norm_1: tensor(127.3156, device='cuda:0') (tensor(168.2277, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.8677 (0.82302943265663) + | > loader_time: 0.0133 (0.011077639441820065) + + + --> TIME: 2024-06-25 03:01:48 -- STEP: 184/591 -- GLOBAL_STEP: 775 + | > loss_disc: 2.5312178134918213 (2.543283536382344) + | > loss_disc_real_0: 0.0978941023349762 (0.20781550852014966) + | > loss_disc_real_1: 0.25076213479042053 (0.23859425407388937) + | > loss_disc_real_2: 0.1917213797569275 (0.23461983717330123) + | > loss_disc_real_3: 0.2951502203941345 (0.23334710121802663) + | > loss_disc_real_4: 0.2825746238231659 (0.22375174680643756) + | > loss_disc_real_5: 0.27114367485046387 (0.21775604675159507) + | > loss_0: 2.5312178134918213 (2.543283536382344) + | > grad_norm_0: tensor(13.2793, device='cuda:0') (tensor(7.8403, device='cuda:0')) + | > loss_gen: 1.770983099937439 (2.15805240428966) + | > loss_kl: 1.9994300603866577 (1.9871719246325286) + | > loss_feat: 3.0389657020568848 (3.0441057254438815) + | > loss_mel: 35.21182632446289 (36.31204030824745) + | > loss_duration: 1.5811386108398438 (1.573320460708245) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 43.60234451293945 (45.07469127489177) + | > grad_norm_1: tensor(99.7790, device='cuda:0') (tensor(163.9294, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.8603 (0.8283557373544445) + | > loader_time: 0.012 (0.011231415945550674) + + + --> TIME: 2024-06-25 03:02:11 -- STEP: 209/591 -- GLOBAL_STEP: 800 + | > loss_disc: 2.95127010345459 (2.548938837918368) + | > loss_disc_real_0: 0.35589468479156494 (0.2095190729036856) + | > loss_disc_real_1: 0.24710974097251892 (0.23874690444275523) + | > loss_disc_real_2: 0.2640831470489502 (0.2338488528317812) + | > loss_disc_real_3: 0.32862332463264465 (0.23310931632963663) + | > loss_disc_real_4: 0.33259305357933044 (0.22382386759565207) + | > loss_disc_real_5: 0.24873016774654388 (0.21870913734418923) + | > loss_0: 2.95127010345459 (2.548938837918368) + | > grad_norm_0: tensor(11.2876, device='cuda:0') (tensor(8.0480, device='cuda:0')) + | > loss_gen: 2.145951986312866 (2.161090459549827) + | > loss_kl: 2.028130531311035 (1.9860516832205668) + | > loss_feat: 2.440887689590454 (3.042405941839994) + | > loss_mel: 36.16078186035156 (36.20688147293896) + | > loss_duration: 1.5515403747558594 (1.573424326175708) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 44.32728958129883 (44.96985432748022) + | > grad_norm_1: tensor(193.4084, device='cuda:0') (tensor(160.4916, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.8659 (0.8353824661108866) + | > loader_time: 0.0125 (0.011445656917882312) + + + --> TIME: 2024-06-25 03:02:33 -- STEP: 234/591 -- GLOBAL_STEP: 825 + | > loss_disc: 2.4236931800842285 (2.5446798088204146) + | > loss_disc_real_0: 0.14012882113456726 (0.20854758738707274) + | > loss_disc_real_1: 0.2474907636642456 (0.23751229646368924) + | > loss_disc_real_2: 0.19239115715026855 (0.23260825503076243) + | > loss_disc_real_3: 0.2222120314836502 (0.23350511064641497) + | > loss_disc_real_4: 0.18920810520648956 (0.22513713353337386) + | > loss_disc_real_5: 0.2628449499607086 (0.21794330809488255) + | > loss_0: 2.4236931800842285 (2.5446798088204146) + | > grad_norm_0: tensor(7.8444, device='cuda:0') (tensor(8.0271, device='cuda:0')) + | > loss_gen: 1.9507557153701782 (2.1619039804507523) + | > loss_kl: 1.9642717838287354 (1.9839003768741574) + | > loss_feat: 3.252593994140625 (3.052549068744366) + | > loss_mel: 35.750213623046875 (36.13743519579246) + | > loss_duration: 1.5920220613479614 (1.5739158121948569) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 44.509857177734375 (44.909704844156934) + | > grad_norm_1: tensor(80.6839, device='cuda:0') (tensor(157.5065, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.8826 (0.8399998204320924) + | > loader_time: 0.0133 (0.011724395629687192) + + + --> TIME: 2024-06-25 03:02:56 -- STEP: 259/591 -- GLOBAL_STEP: 850 + | > loss_disc: 2.4849259853363037 (2.5391766141280243) + | > loss_disc_real_0: 0.2841562032699585 (0.208867928092314) + | > loss_disc_real_1: 0.20907631516456604 (0.2356053569487163) + | > loss_disc_real_2: 0.23820172250270844 (0.2316305233474864) + | > loss_disc_real_3: 0.31361880898475647 (0.23352856186135856) + | > loss_disc_real_4: 0.29084712266921997 (0.2243835830262729) + | > loss_disc_real_5: 0.16002172231674194 (0.21672044403879798) + | > loss_0: 2.4849259853363037 (2.5391766141280243) + | > grad_norm_0: tensor(10.4558, device='cuda:0') (tensor(8.0644, device='cuda:0')) + | > loss_gen: 2.316904067993164 (2.1639775470416973) + | > loss_kl: 2.068270444869995 (1.982015000347005) + | > loss_feat: 3.2032976150512695 (3.0656366725685973) + | > loss_mel: 35.307498931884766 (36.02948906799084) + | > loss_duration: 1.610216736793518 (1.5741828459110039) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 44.50619125366211 (44.81530146064908) + | > grad_norm_1: tensor(129.3454, device='cuda:0') (tensor(155.6091, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.896 (0.8453773577701171) + | > loader_time: 0.0134 (0.012002658659887134) + + + --> TIME: 2024-06-25 03:03:19 -- STEP: 284/591 -- GLOBAL_STEP: 875 + | > loss_disc: 2.47314453125 (2.533077640432707) + | > loss_disc_real_0: 0.13055671751499176 (0.20840349494361537) + | > loss_disc_real_1: 0.22195221483707428 (0.23448359913809194) + | > loss_disc_real_2: 0.24423551559448242 (0.23083757142155942) + | > loss_disc_real_3: 0.22593453526496887 (0.23393031393348332) + | > loss_disc_real_4: 0.2550458312034607 (0.22408518785427153) + | > loss_disc_real_5: 0.21050161123275757 (0.2164876196848255) + | > loss_0: 2.47314453125 (2.533077640432707) + | > grad_norm_0: tensor(8.9504, device='cuda:0') (tensor(8.1176, device='cuda:0')) + | > loss_gen: 1.7124325037002563 (2.1662103710040235) + | > loss_kl: 1.976208209991455 (1.9846728120891142) + | > loss_feat: 2.9175612926483154 (3.0819421704386327) + | > loss_mel: 34.07231140136719 (35.933360959442595) + | > loss_duration: 1.6046783924102783 (1.575601362426516) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 42.2831916809082 (44.741787977621605) + | > grad_norm_1: tensor(156.7640, device='cuda:0') (tensor(155.2412, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.9098 (0.8501462751710918) + | > loader_time: 0.0164 (0.012284829582966554) + + + --> TIME: 2024-06-25 03:03:43 -- STEP: 309/591 -- GLOBAL_STEP: 900 + | > loss_disc: 2.522467613220215 (2.5275166335615147) + | > loss_disc_real_0: 0.25549930334091187 (0.20877686113987154) + | > loss_disc_real_1: 0.2478674203157425 (0.23252317140596193) + | > loss_disc_real_2: 0.24450503289699554 (0.23042820090229071) + | > loss_disc_real_3: 0.19694164395332336 (0.23349882045031364) + | > loss_disc_real_4: 0.20043475925922394 (0.2237871166041368) + | > loss_disc_real_5: 0.2267439365386963 (0.21584232930323072) + | > loss_0: 2.522467613220215 (2.5275166335615147) + | > grad_norm_0: tensor(5.0692, device='cuda:0') (tensor(8.2711, device='cuda:0')) + | > loss_gen: 2.311396360397339 (2.1723066552943) + | > loss_kl: 1.9477781057357788 (1.9862897426950894) + | > loss_feat: 3.267213821411133 (3.101358901335583) + | > loss_mel: 34.51911163330078 (35.83828578566272) + | > loss_duration: 1.610292911529541 (1.5768854899699636) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 43.65579605102539 (44.67512689979333) + | > grad_norm_1: tensor(158.5434, device='cuda:0') (tensor(153.6089, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.9152 (0.8554774796692685) + | > loader_time: 0.0135 (0.012520711398819124) + + + --> TIME: 2024-06-25 03:04:06 -- STEP: 334/591 -- GLOBAL_STEP: 925 + | > loss_disc: 2.4856996536254883 (2.5248759235450615) + | > loss_disc_real_0: 0.3439370393753052 (0.20932582897429694) + | > loss_disc_real_1: 0.23669806122779846 (0.23133989548433323) + | > loss_disc_real_2: 0.30183959007263184 (0.22964812868726467) + | > loss_disc_real_3: 0.30742335319519043 (0.23390630382799102) + | > loss_disc_real_4: 0.24084019660949707 (0.22418859009881933) + | > loss_disc_real_5: 0.3178395628929138 (0.2158406361000624) + | > loss_0: 2.4856996536254883 (2.5248759235450615) + | > grad_norm_0: tensor(17.8396, device='cuda:0') (tensor(8.2811, device='cuda:0')) + | > loss_gen: 1.8218281269073486 (2.174962524525418) + | > loss_kl: 1.998216152191162 (1.986812121853857) + | > loss_feat: 3.3224003314971924 (3.11616840548144) + | > loss_mel: 34.646854400634766 (35.75460806292688) + | > loss_duration: 1.598151445388794 (1.578230516639298) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 43.387451171875 (44.61078193801608) + | > grad_norm_1: tensor(84.0892, device='cuda:0') (tensor(151.1324, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.935 (0.8605527756456843) + | > loader_time: 0.0171 (0.012762718571874207) + + + --> TIME: 2024-06-25 03:04:30 -- STEP: 359/591 -- GLOBAL_STEP: 950 + | > loss_disc: 2.4108850955963135 (2.522004628911989) + | > loss_disc_real_0: 0.17340490221977234 (0.21004084680322818) + | > loss_disc_real_1: 0.21008285880088806 (0.2303695409327828) + | > loss_disc_real_2: 0.1854143589735031 (0.22892367823210266) + | > loss_disc_real_3: 0.27114686369895935 (0.23406470656228928) + | > loss_disc_real_4: 0.15628618001937866 (0.2235758832362701) + | > loss_disc_real_5: 0.23022706806659698 (0.21563248035386418) + | > loss_0: 2.4108850955963135 (2.522004628911989) + | > grad_norm_0: tensor(7.9297, device='cuda:0') (tensor(8.4336, device='cuda:0')) + | > loss_gen: 2.0509562492370605 (2.1793196048577173) + | > loss_kl: 1.941939353942871 (1.9879924424179418) + | > loss_feat: 3.3664684295654297 (3.134671788361742) + | > loss_mel: 33.87992477416992 (35.66491024474246) + | > loss_duration: 1.594991683959961 (1.5801936726716233) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 42.83428192138672 (44.54708803862251) + | > grad_norm_1: tensor(137.1284, device='cuda:0') (tensor(151.7853, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.948 (0.8658181179864824) + | > loader_time: 0.0168 (0.013004593862464511) + + + --> TIME: 2024-06-25 03:04:55 -- STEP: 384/591 -- GLOBAL_STEP: 975 + | > loss_disc: 2.680492877960205 (2.5179673377424465) + | > loss_disc_real_0: 0.1911441832780838 (0.21049529420755184) + | > loss_disc_real_1: 0.17774952948093414 (0.22895618101271487) + | > loss_disc_real_2: 0.1812676191329956 (0.22795788132740805) + | > loss_disc_real_3: 0.20217689871788025 (0.23403723468072712) + | > loss_disc_real_4: 0.19521336257457733 (0.22296294998765612) + | > loss_disc_real_5: 0.15430594980716705 (0.21546926525964716) + | > loss_0: 2.680492877960205 (2.5179673377424465) + | > grad_norm_0: tensor(10.1944, device='cuda:0') (tensor(8.3938, device='cuda:0')) + | > loss_gen: 2.66676926612854 (2.1825806759297848) + | > loss_kl: 1.9078346490859985 (1.988004906103015) + | > loss_feat: 3.1278913021087646 (3.1472746624300867) + | > loss_mel: 35.00467300415039 (35.560774942239114) + | > loss_duration: 1.6432256698608398 (1.582669156603515) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 44.35039520263672 (44.461304624875396) + | > grad_norm_1: tensor(166.6389, device='cuda:0') (tensor(151.1066, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.9736 (0.8717031454046568) + | > loader_time: 0.0182 (0.013290365537007659) + + + --> TIME: 2024-06-25 03:05:19 -- STEP: 409/591 -- GLOBAL_STEP: 1000 + | > loss_disc: 2.423471450805664 (2.515183599187574) + | > loss_disc_real_0: 0.14095120131969452 (0.2108453895286418) + | > loss_disc_real_1: 0.13947951793670654 (0.22786501621995983) + | > loss_disc_real_2: 0.167215496301651 (0.22774964417979887) + | > loss_disc_real_3: 0.20135197043418884 (0.23427491203351242) + | > loss_disc_real_4: 0.1843303143978119 (0.223023845925745) + | > loss_disc_real_5: 0.18524397909641266 (0.215511509597593) + | > loss_0: 2.423471450805664 (2.515183599187574) + | > grad_norm_0: tensor(10.7189, device='cuda:0') (tensor(8.4675, device='cuda:0')) + | > loss_gen: 2.619412899017334 (2.187121774864665) + | > loss_kl: 2.0430221557617188 (1.9891071363299866) + | > loss_feat: 3.502229928970337 (3.164826944579704) + | > loss_mel: 34.85422897338867 (35.48039277316888) + | > loss_duration: 1.5815002918243408 (1.5842841102907883) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 44.60039520263672 (44.405733001261304) + | > grad_norm_1: tensor(117.1476, device='cuda:0') (tensor(149.6326, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.9768 (0.8775204300588967) + | > loader_time: 0.0164 (0.013526029866598645) + + + --> TIME: 2024-06-25 03:05:44 -- STEP: 434/591 -- GLOBAL_STEP: 1025 + | > loss_disc: 2.377248764038086 (2.510381308019434) + | > loss_disc_real_0: 0.23853585124015808 (0.2111940458931956) + | > loss_disc_real_1: 0.1569882184267044 (0.22701442035661862) + | > loss_disc_real_2: 0.19629228115081787 (0.22757054041332914) + | > loss_disc_real_3: 0.2839317321777344 (0.23438600958904363) + | > loss_disc_real_4: 0.19771841168403625 (0.22264734307123768) + | > loss_disc_real_5: 0.16731350123882294 (0.2151620961931719) + | > loss_0: 2.377248764038086 (2.510381308019434) + | > grad_norm_0: tensor(6.7366, device='cuda:0') (tensor(8.5213, device='cuda:0')) + | > loss_gen: 2.213822364807129 (2.188584486460359) + | > loss_kl: 1.996525526046753 (1.9891506480181822) + | > loss_feat: 3.5424182415008545 (3.1785971629454783) + | > loss_mel: 33.571468353271484 (35.383197010936804) + | > loss_duration: 1.598149061203003 (1.5859413770486679) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 42.92238235473633 (44.325470911193015) + | > grad_norm_1: tensor(131.4084, device='cuda:0') (tensor(149.4415, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 0.9973 (0.8838683372269028) + | > loader_time: 0.0168 (0.01378756624213013) + + + --> TIME: 2024-06-25 03:06:11 -- STEP: 459/591 -- GLOBAL_STEP: 1050 + | > loss_disc: 2.5295662879943848 (2.507722133644784) + | > loss_disc_real_0: 0.17815294861793518 (0.21135329687257737) + | > loss_disc_real_1: 0.18566137552261353 (0.22629654271150723) + | > loss_disc_real_2: 0.22099977731704712 (0.22715304986713758) + | > loss_disc_real_3: 0.22784653306007385 (0.23399793676431402) + | > loss_disc_real_4: 0.21647846698760986 (0.22256336231312202) + | > loss_disc_real_5: 0.354655921459198 (0.21574616232533864) + | > loss_0: 2.5295662879943848 (2.507722133644784) + | > grad_norm_0: tensor(9.4212, device='cuda:0') (tensor(8.5255, device='cuda:0')) + | > loss_gen: 2.1121327877044678 (2.189978971200834) + | > loss_kl: 1.9935007095336914 (1.9895703486108052) + | > loss_feat: 3.4355342388153076 (3.1897538212130003) + | > loss_mel: 34.188453674316406 (35.287388147092315) + | > loss_duration: 1.6221789121627808 (1.587936135418794) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 43.35179901123047 (44.24462760975159) + | > grad_norm_1: tensor(102.1629, device='cuda:0') (tensor(148.8732, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 1.0277 (0.8913040239047381) + | > loader_time: 0.0183 (0.014072342895474576) + + + --> TIME: 2024-06-25 03:06:37 -- STEP: 484/591 -- GLOBAL_STEP: 1075 + | > loss_disc: 2.894408702850342 (2.5164437959016834) + | > loss_disc_real_0: 0.2650354206562042 (0.21249081776284975) + | > loss_disc_real_1: 0.29896795749664307 (0.22736474272818108) + | > loss_disc_real_2: 0.23582403361797333 (0.228186826213948) + | > loss_disc_real_3: 0.19774290919303894 (0.23429078300876066) + | > loss_disc_real_4: 0.21218423545360565 (0.2239783283877225) + | > loss_disc_real_5: 0.3585188388824463 (0.21707929606156903) + | > loss_0: 2.894408702850342 (2.5164437959016834) + | > grad_norm_0: tensor(9.2423, device='cuda:0') (tensor(8.7730, device='cuda:0')) + | > loss_gen: 1.842199444770813 (2.1997689055509833) + | > loss_kl: 2.0572614669799805 (1.990720588806247) + | > loss_feat: 2.70078182220459 (3.2038858577239604) + | > loss_mel: 34.415184020996094 (35.23836377829557) + | > loss_duration: 1.6314244270324707 (1.5903394776434936) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 42.6468505859375 (44.223078775011814) + | > grad_norm_1: tensor(101.1536, device='cuda:0') (tensor(148.0931, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 1.0242 (0.898912787437439) + | > loader_time: 0.0224 (0.01436650802281278) + + + --> TIME: 2024-06-25 03:07:05 -- STEP: 509/591 -- GLOBAL_STEP: 1100 + | > loss_disc: 2.7139463424682617 (2.529914536972644) + | > loss_disc_real_0: 0.23823398351669312 (0.21297336444405066) + | > loss_disc_real_1: 0.15285344421863556 (0.22855534228864022) + | > loss_disc_real_2: 0.3366057574748993 (0.23126698411225335) + | > loss_disc_real_3: 0.2072255164384842 (0.23379973004515373) + | > loss_disc_real_4: 0.3729293942451477 (0.22702401517826354) + | > loss_disc_real_5: 0.32972875237464905 (0.21868250151631408) + | > loss_0: 2.7139463424682617 (2.529914536972644) + | > grad_norm_0: tensor(6.5888, device='cuda:0') (tensor(8.8324, device='cuda:0')) + | > loss_gen: 2.077491044998169 (2.206297114930596) + | > loss_kl: 1.9483715295791626 (1.993222983974839) + | > loss_feat: 3.1458559036254883 (3.2020951839701826) + | > loss_mel: 34.664222717285156 (35.19029877368498) + | > loss_duration: 1.6613106727600098 (1.5927054640820544) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 43.49725341796875 (44.184619686224146) + | > grad_norm_1: tensor(172.0930, device='cuda:0') (tensor(146.9988, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 1.0637 (0.9077772325991648) + | > loader_time: 0.0212 (0.01468542376295289) + + + --> TIME: 2024-06-25 03:07:34 -- STEP: 534/591 -- GLOBAL_STEP: 1125 + | > loss_disc: 2.6714794635772705 (2.5371540869666385) + | > loss_disc_real_0: 0.25672921538352966 (0.2130380373043514) + | > loss_disc_real_1: 0.2701514959335327 (0.22989628899298353) + | > loss_disc_real_2: 0.2170010507106781 (0.23131141012136855) + | > loss_disc_real_3: 0.2255493402481079 (0.23628331473173944) + | > loss_disc_real_4: 0.2692352533340454 (0.22783910763565074) + | > loss_disc_real_5: 0.2575705349445343 (0.2191575278671047) + | > loss_0: 2.6714794635772705 (2.5371540869666385) + | > grad_norm_0: tensor(4.5267, device='cuda:0') (tensor(8.7953, device='cuda:0')) + | > loss_gen: 2.067467212677002 (2.2078096453616705) + | > loss_kl: 1.999019742012024 (1.9929104342442774) + | > loss_feat: 2.763270378112793 (3.1923409584309717) + | > loss_mel: 32.7255744934082 (35.13023294313122) + | > loss_duration: 1.6359736919403076 (1.5954339339045545) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 41.191307067871094 (44.11872808049235) + | > grad_norm_1: tensor(97.6535, device='cuda:0') (tensor(145.2387, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 1.1487 (0.9183085897888584) + | > loader_time: 0.0253 (0.014999871396839849) + + + --> TIME: 2024-06-25 03:08:03 -- STEP: 559/591 -- GLOBAL_STEP: 1150 + | > loss_disc: 2.5325307846069336 (2.536360835346637) + | > loss_disc_real_0: 0.16641581058502197 (0.21347875731842247) + | > loss_disc_real_1: 0.25598424673080444 (0.2292287998527875) + | > loss_disc_real_2: 0.26187318563461304 (0.2307550569259843) + | > loss_disc_real_3: 0.25991126894950867 (0.23607293752976735) + | > loss_disc_real_4: 0.2466498166322708 (0.22844614631001553) + | > loss_disc_real_5: 0.20893168449401855 (0.2188082380830069) + | > loss_0: 2.5325307846069336 (2.536360835346637) + | > grad_norm_0: tensor(7.8233, device='cuda:0') (tensor(8.7340, device='cuda:0')) + | > loss_gen: 1.8966968059539795 (2.2026530203537793) + | > loss_kl: 2.0433948040008545 (1.9954541326635427) + | > loss_feat: 2.9389071464538574 (3.1832333033327966) + | > loss_mel: 32.28898620605469 (35.02995290739163) + | > loss_duration: 1.6605651378631592 (1.5981726691206792) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 40.828548431396484 (44.009466167852594) + | > grad_norm_1: tensor(185.9949, device='cuda:0') (tensor(145.8950, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 1.1494 (0.9291023154591407) + | > loader_time: 0.0198 (0.01532163005822034) + + + --> TIME: 2024-06-25 03:08:35 -- STEP: 584/591 -- GLOBAL_STEP: 1175 + | > loss_disc: 2.6202919483184814 (2.5332926687312445) + | > loss_disc_real_0: 0.23866479098796844 (0.21339013766854592) + | > loss_disc_real_1: 0.2820347547531128 (0.22825360323672425) + | > loss_disc_real_2: 0.29516831040382385 (0.23029557193233952) + | > loss_disc_real_3: 0.3427969813346863 (0.2357379333626428) + | > loss_disc_real_4: 0.20043422281742096 (0.22826914516060728) + | > loss_disc_real_5: 0.3074789047241211 (0.21882609351959134) + | > loss_0: 2.6202919483184814 (2.5332926687312445) + | > grad_norm_0: tensor(9.1563, device='cuda:0') (tensor(8.6651, device='cuda:0')) + | > loss_gen: 2.2202048301696777 (2.2008840545807806) + | > loss_kl: 2.044218063354492 (1.9972305401955566) + | > loss_feat: 2.9077341556549072 (3.1843326589832563) + | > loss_mel: 32.90556716918945 (34.9299988420042) + | > loss_duration: 1.6575853824615479 (1.6007066362932927) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 41.73530960083008 (43.913152864534595) + | > grad_norm_1: tensor(182.6417, device='cuda:0') (tensor(146.5490, device='cuda:0')) + | > current_lr_0: 0.000199975 + | > current_lr_1: 0.000199975 + | > step_time: 1.3292 (0.9424843494206261) + | > loader_time: 0.0223 (0.015711432858689197) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037542150375690867 (-0.00010769417945374806) + | > avg_loss_disc: 2.577631138740702 (-0.006989453701262693) + | > avg_loss_disc_real_0: 0.17637559169150413 (-0.03579020880638284) + | > avg_loss_disc_real_1: 0.26676153026996774 (+0.05449855581242988) + | > avg_loss_disc_real_2: 0.20470569853452927 (-0.012329608360503563) + | > avg_loss_disc_real_3: 0.22298222399772483 (+0.038161661079589376) + | > avg_loss_disc_real_4: 0.22061868392406625 (+0.04992743898579413) + | > avg_loss_disc_real_5: 0.18215589748418076 (+0.03910153509771569) + | > avg_loss_0: 2.577631138740702 (-0.006989453701262693) + | > avg_loss_gen: 2.0537032675235833 (+0.2908143819646636) + | > avg_loss_kl: 2.2996217316769547 (+0.16102866923555537) + | > avg_loss_feat: 3.465375545177054 (+0.6637694176207201) + | > avg_loss_mel: 33.05276197068235 (-3.848825576457571) + | > avg_loss_duration: 1.6495573825024543 (+0.06941264233690614) + | > avg_loss_1: 42.521019712407536 (-2.6638009091641024) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_1182.pth + + > EPOCH: 3/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 03:08:56)  + + --> TIME: 2024-06-25 03:09:12 -- STEP: 18/591 -- GLOBAL_STEP: 1200 + | > loss_disc: 2.6473734378814697 (2.5249469810061984) + | > loss_disc_real_0: 0.16226878762245178 (0.2255612595213784) + | > loss_disc_real_1: 0.1989288330078125 (0.21080035550726783) + | > loss_disc_real_2: 0.2324453741312027 (0.22847191037403214) + | > loss_disc_real_3: 0.22839632630348206 (0.2458630676070849) + | > loss_disc_real_4: 0.25384798645973206 (0.2377534843981266) + | > loss_disc_real_5: 0.1944894939661026 (0.22689019557502535) + | > loss_0: 2.6473734378814697 (2.5249469810061984) + | > grad_norm_0: tensor(10.7483, device='cuda:0') (tensor(11.0262, device='cuda:0')) + | > loss_gen: 2.4983420372009277 (2.3111048539479575) + | > loss_kl: 2.242842197418213 (2.282841351297167) + | > loss_feat: 3.265030860900879 (3.6249090830485025) + | > loss_mel: 33.49146270751953 (33.5480195151435) + | > loss_duration: 1.5487606525421143 (1.5204187830289204) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 43.04643630981445 (43.28729438781738) + | > grad_norm_1: tensor(198.3113, device='cuda:0') (tensor(144.6449, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.7758 (0.7761239210764567) + | > loader_time: 0.0092 (0.008663045035468208) + + + --> TIME: 2024-06-25 03:09:33 -- STEP: 43/591 -- GLOBAL_STEP: 1225 + | > loss_disc: 2.420368194580078 (2.5035691483076232) + | > loss_disc_real_0: 0.3133714199066162 (0.22517557875361555) + | > loss_disc_real_1: 0.2201237827539444 (0.20986735508885496) + | > loss_disc_real_2: 0.17362777888774872 (0.22255368548077206) + | > loss_disc_real_3: 0.274418443441391 (0.2459768143504165) + | > loss_disc_real_4: 0.18385210633277893 (0.23142084739235944) + | > loss_disc_real_5: 0.1666860282421112 (0.23067391403885776) + | > loss_0: 2.420368194580078 (2.5035691483076232) + | > grad_norm_0: tensor(10.3956, device='cuda:0') (tensor(10.4109, device='cuda:0')) + | > loss_gen: 1.9859931468963623 (2.257308264111363) + | > loss_kl: 2.1568896770477295 (2.23063231900681) + | > loss_feat: 3.346108913421631 (3.5392319657081783) + | > loss_mel: 32.292640686035156 (33.11198194636854) + | > loss_duration: 1.6238625049591064 (1.5481844003810439) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 41.405494689941406 (42.68733916171762) + | > grad_norm_1: tensor(229.2561, device='cuda:0') (tensor(141.5537, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.8099 (0.7900653883468273) + | > loader_time: 0.0115 (0.009383916854858397) + + + --> TIME: 2024-06-25 03:09:53 -- STEP: 68/591 -- GLOBAL_STEP: 1250 + | > loss_disc: 2.2944726943969727 (2.498420298099518) + | > loss_disc_real_0: 0.1516551971435547 (0.22113040473093004) + | > loss_disc_real_1: 0.1867869645357132 (0.20800121172386057) + | > loss_disc_real_2: 0.20968037843704224 (0.22424744124359944) + | > loss_disc_real_3: 0.20139756798744202 (0.24175407167743235) + | > loss_disc_real_4: 0.16792219877243042 (0.2282609230674365) + | > loss_disc_real_5: 0.20585721731185913 (0.22445732771473773) + | > loss_0: 2.2944726943969727 (2.498420298099518) + | > grad_norm_0: tensor(10.1643, device='cuda:0') (tensor(8.9269, device='cuda:0')) + | > loss_gen: 2.0469818115234375 (2.2297131138689377) + | > loss_kl: 2.2157344818115234 (2.2080606940914604) + | > loss_feat: 3.751749038696289 (3.5056120122180268) + | > loss_mel: 33.3154182434082 (32.804381342495184) + | > loss_duration: 1.6031506061553955 (1.5650888523634743) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 42.93303298950195 (42.31285611320944) + | > grad_norm_1: tensor(62.8659, device='cuda:0') (tensor(142.2379, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.8117 (0.7992338327800527) + | > loader_time: 0.0105 (0.009833910885979147) + + + --> TIME: 2024-06-25 03:10:15 -- STEP: 93/591 -- GLOBAL_STEP: 1275 + | > loss_disc: 2.8254852294921875 (2.51568736824938) + | > loss_disc_real_0: 0.23853251338005066 (0.2217207555008191) + | > loss_disc_real_1: 0.256756991147995 (0.21047063139818048) + | > loss_disc_real_2: 0.3063828945159912 (0.2274403761952154) + | > loss_disc_real_3: 0.22559383511543274 (0.23871505004103466) + | > loss_disc_real_4: 0.2051440179347992 (0.23069884483852693) + | > loss_disc_real_5: 0.22112886607646942 (0.2253477031825691) + | > loss_0: 2.8254852294921875 (2.51568736824938) + | > grad_norm_0: tensor(7.8585, device='cuda:0') (tensor(8.5895, device='cuda:0')) + | > loss_gen: 2.1472513675689697 (2.2351982439717943) + | > loss_kl: 2.189890146255493 (2.2031111448041845) + | > loss_feat: 2.996105194091797 (3.485121019424931) + | > loss_mel: 32.35627746582031 (32.71455494050057) + | > loss_duration: 1.6406716108322144 (1.5779126792825682) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 41.330196380615234 (42.21589812412058) + | > grad_norm_1: tensor(155.0035, device='cuda:0') (tensor(138.0182, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.8694 (0.8081448898520522) + | > loader_time: 0.0122 (0.010267985764370168) + + + --> TIME: 2024-06-25 03:10:36 -- STEP: 118/591 -- GLOBAL_STEP: 1300 + | > loss_disc: 2.435605764389038 (2.5054334705158814) + | > loss_disc_real_0: 0.11489614099264145 (0.22063560973284607) + | > loss_disc_real_1: 0.2406126707792282 (0.2089631115481005) + | > loss_disc_real_2: 0.21233335137367249 (0.22531528714097152) + | > loss_disc_real_3: 0.20301491022109985 (0.23792203439999435) + | > loss_disc_real_4: 0.2589678466320038 (0.229086125490524) + | > loss_disc_real_5: 0.2618181109428406 (0.224653278114432) + | > loss_0: 2.435605764389038 (2.5054334705158814) + | > grad_norm_0: tensor(14.2302, device='cuda:0') (tensor(8.2647, device='cuda:0')) + | > loss_gen: 2.1005797386169434 (2.216608250545243) + | > loss_kl: 2.2075488567352295 (2.1993784550893087) + | > loss_feat: 3.525674343109131 (3.463919548665063) + | > loss_mel: 31.038820266723633 (32.59671167599953) + | > loss_duration: 1.621437907218933 (1.583799821845556) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 40.494056701660156 (42.060417724867996) + | > grad_norm_1: tensor(81.4750, device='cuda:0') (tensor(135.4372, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.8292 (0.8146335856389192) + | > loader_time: 0.0106 (0.010523111133252157) + + + --> TIME: 2024-06-25 03:10:57 -- STEP: 143/591 -- GLOBAL_STEP: 1325 + | > loss_disc: 2.56943941116333 (2.5141291835091333) + | > loss_disc_real_0: 0.2640005350112915 (0.22187475469354148) + | > loss_disc_real_1: 0.16307158768177032 (0.20964087587553304) + | > loss_disc_real_2: 0.19353927671909332 (0.2264692074248007) + | > loss_disc_real_3: 0.2878777086734772 (0.23836609335629255) + | > loss_disc_real_4: 0.3010925352573395 (0.23085384503319548) + | > loss_disc_real_5: 0.30130627751350403 (0.22568930831405667) + | > loss_0: 2.56943941116333 (2.5141291835091333) + | > grad_norm_0: tensor(7.0620, device='cuda:0') (tensor(8.4435, device='cuda:0')) + | > loss_gen: 1.6184499263763428 (2.2108052352091647) + | > loss_kl: 2.042660713195801 (2.200094477280036) + | > loss_feat: 3.1540956497192383 (3.447352089248337) + | > loss_mel: 32.376976013183594 (32.534638784982086) + | > loss_duration: 1.5757334232330322 (1.5857890169103666) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 40.767913818359375 (41.978679683658626) + | > grad_norm_1: tensor(116.9604, device='cuda:0') (tensor(136.4726, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.8414 (0.8197428796674823) + | > loader_time: 0.0127 (0.010701791389838797) + + + --> TIME: 2024-06-25 03:11:19 -- STEP: 168/591 -- GLOBAL_STEP: 1350 + | > loss_disc: 2.382291555404663 (2.5104084610939035) + | > loss_disc_real_0: 0.15473562479019165 (0.21967061436069862) + | > loss_disc_real_1: 0.20922525227069855 (0.20950508915952273) + | > loss_disc_real_2: 0.1547829955816269 (0.22618716471784173) + | > loss_disc_real_3: 0.23443740606307983 (0.23631762420492514) + | > loss_disc_real_4: 0.18594036996364594 (0.22979949525601806) + | > loss_disc_real_5: 0.19085334241390228 (0.22369000466451758) + | > loss_0: 2.382291555404663 (2.5104084610939035) + | > grad_norm_0: tensor(8.0712, device='cuda:0') (tensor(8.3829, device='cuda:0')) + | > loss_gen: 2.260587453842163 (2.203934627629461) + | > loss_kl: 2.350630283355713 (2.2026854178735173) + | > loss_feat: 3.474846839904785 (3.432744275955927) + | > loss_mel: 32.805145263671875 (32.471136433737605) + | > loss_duration: 1.5713655948638916 (1.5851408228987742) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 42.46257400512695 (41.89564166750226) + | > grad_norm_1: tensor(88.3892, device='cuda:0') (tensor(133.9460, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.8766 (0.826110946280616) + | > loader_time: 0.0138 (0.010920337268284388) + + + --> TIME: 2024-06-25 03:11:42 -- STEP: 193/591 -- GLOBAL_STEP: 1375 + | > loss_disc: 2.4502124786376953 (2.5063962788161853) + | > loss_disc_real_0: 0.16027067601680756 (0.21854120589445292) + | > loss_disc_real_1: 0.17600733041763306 (0.2099651777682527) + | > loss_disc_real_2: 0.15831972658634186 (0.22569640455622747) + | > loss_disc_real_3: 0.16247788071632385 (0.23537837015223625) + | > loss_disc_real_4: 0.1792648732662201 (0.2287453462808861) + | > loss_disc_real_5: 0.1399429440498352 (0.2224297912627304) + | > loss_0: 2.4502124786376953 (2.5063962788161853) + | > grad_norm_0: tensor(12.3423, device='cuda:0') (tensor(8.3175, device='cuda:0')) + | > loss_gen: 2.6142048835754395 (2.2000455146008826) + | > loss_kl: 2.1322481632232666 (2.2044089402559495) + | > loss_feat: 3.7371253967285156 (3.4218202860244196) + | > loss_mel: 32.28452682495117 (32.40574698374061) + | > loss_duration: 1.5422033071517944 (1.584307031927949) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 42.310306549072266 (41.81632880966898) + | > grad_norm_1: tensor(115.0668, device='cuda:0') (tensor(134.7522, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.8717 (0.8325390655142041) + | > loader_time: 0.0145 (0.011139506503090339) + + + --> TIME: 2024-06-25 03:12:04 -- STEP: 218/591 -- GLOBAL_STEP: 1400 + | > loss_disc: 2.584567070007324 (2.509733871582452) + | > loss_disc_real_0: 0.2641634941101074 (0.21841630961643446) + | > loss_disc_real_1: 0.18895269930362701 (0.21038683073236308) + | > loss_disc_real_2: 0.19211632013320923 (0.22626377132917763) + | > loss_disc_real_3: 0.2547622323036194 (0.23627886982685928) + | > loss_disc_real_4: 0.30489951372146606 (0.22982359305024147) + | > loss_disc_real_5: 0.28064313530921936 (0.2231942223162826) + | > loss_0: 2.584567070007324 (2.509733871582452) + | > grad_norm_0: tensor(8.8562, device='cuda:0') (tensor(8.4669, device='cuda:0')) + | > loss_gen: 1.9989230632781982 (2.1960323086572346) + | > loss_kl: 2.159947395324707 (2.2037704707285664) + | > loss_feat: 2.998562812805176 (3.4152644019608105) + | > loss_mel: 31.63018226623535 (32.33062678520835) + | > loss_duration: 1.581102967262268 (1.584577491524023) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 40.36872100830078 (41.73027152315192) + | > grad_norm_1: tensor(95.1371, device='cuda:0') (tensor(133.7614, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.8903 (0.8375237480216072) + | > loader_time: 0.015 (0.011395003817497043) + + + --> TIME: 2024-06-25 03:12:27 -- STEP: 243/591 -- GLOBAL_STEP: 1425 + | > loss_disc: 2.5540518760681152 (2.5165714560222243) + | > loss_disc_real_0: 0.18061476945877075 (0.21795085909557932) + | > loss_disc_real_1: 0.17477494478225708 (0.21068575874032308) + | > loss_disc_real_2: 0.23324304819107056 (0.2267010232487333) + | > loss_disc_real_3: 0.2746189832687378 (0.23681224526937117) + | > loss_disc_real_4: 0.3037351667881012 (0.23086274979413782) + | > loss_disc_real_5: 0.2705623209476471 (0.2238047566310859) + | > loss_0: 2.5540518760681152 (2.5165714560222243) + | > grad_norm_0: tensor(5.7067, device='cuda:0') (tensor(8.5020, device='cuda:0')) + | > loss_gen: 1.9656190872192383 (2.1913344565732977) + | > loss_kl: 2.3465960025787354 (2.204425897617889) + | > loss_feat: 3.139606237411499 (3.397426413901058) + | > loss_mel: 32.22230911254883 (32.2500834209929) + | > loss_duration: 1.5472211837768555 (1.5853431612375835) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 41.221351623535156 (41.62861331892602) + | > grad_norm_1: tensor(186.3807, device='cuda:0') (tensor(133.7813, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.8821 (0.8423579190493613) + | > loader_time: 0.0145 (0.01167460037357032) + + + --> TIME: 2024-06-25 03:12:50 -- STEP: 268/591 -- GLOBAL_STEP: 1450 + | > loss_disc: 2.4857239723205566 (2.5219843592216726) + | > loss_disc_real_0: 0.2983707785606384 (0.21780937301245198) + | > loss_disc_real_1: 0.2371934950351715 (0.211210511291205) + | > loss_disc_real_2: 0.2874380350112915 (0.22662763856351378) + | > loss_disc_real_3: 0.29169216752052307 (0.2363734342046638) + | > loss_disc_real_4: 0.13990730047225952 (0.23109993586010896) + | > loss_disc_real_5: 0.20384004712104797 (0.22423371261180336) + | > loss_0: 2.4857239723205566 (2.5219843592216726) + | > grad_norm_0: tensor(14.3635, device='cuda:0') (tensor(8.6998, device='cuda:0')) + | > loss_gen: 2.1383655071258545 (2.184468608293961) + | > loss_kl: 2.222106695175171 (2.1984813111041905) + | > loss_feat: 3.5088322162628174 (3.3780472412038205) + | > loss_mel: 31.21506118774414 (32.16998530145902) + | > loss_duration: 1.5770907402038574 (1.5862738134256051) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 40.66145706176758 (41.51725626703519) + | > grad_norm_1: tensor(153.2246, device='cuda:0') (tensor(131.5484, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.9039 (0.8478992198830222) + | > loader_time: 0.0148 (0.011968831517803135) + + + --> TIME: 2024-06-25 03:13:13 -- STEP: 293/591 -- GLOBAL_STEP: 1475 + | > loss_disc: 2.4229319095611572 (2.524756653723864) + | > loss_disc_real_0: 0.15097549557685852 (0.21699390595988607) + | > loss_disc_real_1: 0.2191770225763321 (0.21137557852593708) + | > loss_disc_real_2: 0.26145491003990173 (0.2266514660097634) + | > loss_disc_real_3: 0.24301747977733612 (0.23658689785329148) + | > loss_disc_real_4: 0.22919023036956787 (0.23146600352517574) + | > loss_disc_real_5: 0.22737960517406464 (0.22442720062496718) + | > loss_0: 2.4229319095611572 (2.524756653723864) + | > grad_norm_0: tensor(8.0500, device='cuda:0') (tensor(8.7230, device='cuda:0')) + | > loss_gen: 2.164907455444336 (2.1784407441526557) + | > loss_kl: 2.236386775970459 (2.198660288654496) + | > loss_feat: 3.4747533798217773 (3.3624850637676773) + | > loss_mel: 31.197566986083984 (32.09344429693126) + | > loss_duration: 1.5960133075714111 (1.588295115952606) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 40.66962814331055 (41.4213255013215) + | > grad_norm_1: tensor(83.8861, device='cuda:0') (tensor(131.4407, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.9117 (0.8534415638894352) + | > loader_time: 0.0138 (0.012242100751440676) + + + --> TIME: 2024-06-25 03:13:37 -- STEP: 318/591 -- GLOBAL_STEP: 1500 + | > loss_disc: 2.510281801223755 (2.5291861970469642) + | > loss_disc_real_0: 0.20770850777626038 (0.21702564219540021) + | > loss_disc_real_1: 0.20882906019687653 (0.2115535961475762) + | > loss_disc_real_2: 0.16791841387748718 (0.22669258031923825) + | > loss_disc_real_3: 0.18028895556926727 (0.2370296022708311) + | > loss_disc_real_4: 0.19183382391929626 (0.23187417904817084) + | > loss_disc_real_5: 0.21395103633403778 (0.22452644846817982) + | > loss_0: 2.510281801223755 (2.5291861970469642) + | > grad_norm_0: tensor(6.5122, device='cuda:0') (tensor(8.7680, device='cuda:0')) + | > loss_gen: 1.9327337741851807 (2.1717745725463793) + | > loss_kl: 2.1087141036987305 (2.197322618661436) + | > loss_feat: 2.8521761894226074 (3.3433124119380744) + | > loss_mel: 30.208818435668945 (32.03155877455228) + | > loss_duration: 1.630552053451538 (1.5903551256131834) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 38.732994079589844 (41.334323475195895) + | > grad_norm_1: tensor(85.1101, device='cuda:0') (tensor(132.6523, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.9246 (0.8587663301132008) + | > loader_time: 0.0164 (0.012491403135863482) + + + --> TIME: 2024-06-25 03:14:01 -- STEP: 343/591 -- GLOBAL_STEP: 1525 + | > loss_disc: 2.787036657333374 (2.537151622355159) + | > loss_disc_real_0: 0.1515660136938095 (0.21746612994038328) + | > loss_disc_real_1: 0.3010576367378235 (0.21263556069431083) + | > loss_disc_real_2: 0.3356267213821411 (0.22743662833385503) + | > loss_disc_real_3: 0.3464597463607788 (0.23754708325550092) + | > loss_disc_real_4: 0.3399561047554016 (0.23273659380201694) + | > loss_disc_real_5: 0.3136459290981293 (0.22546090848157427) + | > loss_0: 2.787036657333374 (2.537151622355159) + | > grad_norm_0: tensor(11.6477, device='cuda:0') (tensor(8.8776, device='cuda:0')) + | > loss_gen: 1.9257075786590576 (2.1698277166911564) + | > loss_kl: 2.1812491416931152 (2.1966367925221295) + | > loss_feat: 2.879741668701172 (3.3274032916689067) + | > loss_mel: 31.252397537231445 (31.962561510047156) + | > loss_duration: 1.5659775733947754 (1.5925364737608) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 39.805076599121094 (41.24896579442146) + | > grad_norm_1: tensor(143.1788, device='cuda:0') (tensor(132.9247, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.9284 (0.8644879413415668) + | > loader_time: 0.0149 (0.012756620134626109) + + + --> TIME: 2024-06-25 03:14:25 -- STEP: 368/591 -- GLOBAL_STEP: 1550 + | > loss_disc: 2.5488767623901367 (2.53992018427538) + | > loss_disc_real_0: 0.14616988599300385 (0.21688866202274096) + | > loss_disc_real_1: 0.20201395452022552 (0.21264593521861927) + | > loss_disc_real_2: 0.2396012544631958 (0.22731876047328123) + | > loss_disc_real_3: 0.2842191755771637 (0.23741496281455393) + | > loss_disc_real_4: 0.29824551939964294 (0.2332576060384188) + | > loss_disc_real_5: 0.22958414256572723 (0.2251610271025287) + | > loss_0: 2.5488767623901367 (2.53992018427538) + | > grad_norm_0: tensor(10.9569, device='cuda:0') (tensor(8.8018, device='cuda:0')) + | > loss_gen: 2.0048539638519287 (2.1628508677949094) + | > loss_kl: 2.1645171642303467 (2.19494949961486) + | > loss_feat: 3.219014883041382 (3.308591437080633) + | > loss_mel: 32.454071044921875 (31.928673433220926) + | > loss_duration: 1.6326767206192017 (1.5955031497970877) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 41.47513198852539 (41.19056839528286) + | > grad_norm_1: tensor(176.4376, device='cuda:0') (tensor(132.8486, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.9614 (0.8698611816634301) + | > loader_time: 0.0178 (0.013028445451155946) + + + --> TIME: 2024-06-25 03:14:49 -- STEP: 393/591 -- GLOBAL_STEP: 1575 + | > loss_disc: 2.653871774673462 (2.544312815629801) + | > loss_disc_real_0: 0.1330476850271225 (0.21679455724381308) + | > loss_disc_real_1: 0.22667750716209412 (0.21285793484773952) + | > loss_disc_real_2: 0.24596454203128815 (0.22726866752667593) + | > loss_disc_real_3: 0.2658195495605469 (0.23750357420844886) + | > loss_disc_real_4: 0.32332509756088257 (0.23355972772563688) + | > loss_disc_real_5: 0.31809529662132263 (0.22569135098739435) + | > loss_0: 2.653871774673462 (2.544312815629801) + | > grad_norm_0: tensor(13.8108, device='cuda:0') (tensor(9.0019, device='cuda:0')) + | > loss_gen: 2.0258004665374756 (2.158321516507761) + | > loss_kl: 2.24651837348938 (2.19434670243251) + | > loss_feat: 2.899029493331909 (3.290635315213192) + | > loss_mel: 30.760385513305664 (31.8645236182759) + | > loss_duration: 1.6308534145355225 (1.5978421612853622) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 39.562583923339844 (41.10566934677779) + | > grad_norm_1: tensor(66.7857, device='cuda:0') (tensor(131.9378, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.9761 (0.8759918995485959) + | > loader_time: 0.0172 (0.013284260383392405) + + + --> TIME: 2024-06-25 03:15:15 -- STEP: 418/591 -- GLOBAL_STEP: 1600 + | > loss_disc: 2.588301181793213 (2.546116210627215) + | > loss_disc_real_0: 0.14265602827072144 (0.21614198176341765) + | > loss_disc_real_1: 0.22508898377418518 (0.21324939013383035) + | > loss_disc_real_2: 0.2214284986257553 (0.22709875402339344) + | > loss_disc_real_3: 0.21680444478988647 (0.23726979176108345) + | > loss_disc_real_4: 0.19861073791980743 (0.2333779447850143) + | > loss_disc_real_5: 0.15765127539634705 (0.22557633824682122) + | > loss_0: 2.588301181793213 (2.546116210627215) + | > grad_norm_0: tensor(11.0517, device='cuda:0') (tensor(8.8701, device='cuda:0')) + | > loss_gen: 2.2906649112701416 (2.151978416876362) + | > loss_kl: 2.149651288986206 (2.1939239892662994) + | > loss_feat: 3.045870065689087 (3.2719549088957214) + | > loss_mel: 29.869945526123047 (31.774360451401723) + | > loss_duration: 1.5519343614578247 (1.5999093540547573) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 38.90806579589844 (40.99212714929895) + | > grad_norm_1: tensor(66.9445, device='cuda:0') (tensor(131.5660, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 0.9998 (0.8823618301364222) + | > loader_time: 0.0163 (0.01355264802868856) + + + --> TIME: 2024-06-25 03:15:40 -- STEP: 443/591 -- GLOBAL_STEP: 1625 + | > loss_disc: 2.471174955368042 (2.5479624454377747) + | > loss_disc_real_0: 0.20488403737545013 (0.21623372320195083) + | > loss_disc_real_1: 0.18010465800762177 (0.21291803916743862) + | > loss_disc_real_2: 0.1928735375404358 (0.22759745179194096) + | > loss_disc_real_3: 0.27183863520622253 (0.23730076822551055) + | > loss_disc_real_4: 0.23023973405361176 (0.23323684989666024) + | > loss_disc_real_5: 0.2151675820350647 (0.22576077825665206) + | > loss_0: 2.471174955368042 (2.5479624454377747) + | > grad_norm_0: tensor(5.5410, device='cuda:0') (tensor(9.0544, device='cuda:0')) + | > loss_gen: 2.2608463764190674 (2.148489455068088) + | > loss_kl: 2.3337368965148926 (2.1934094620073887) + | > loss_feat: 3.1633687019348145 (3.258268015648388) + | > loss_mel: 31.037511825561523 (31.698077315939752) + | > loss_duration: 1.68019437789917 (1.6023841302497128) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 40.47565841674805 (40.900628427888634) + | > grad_norm_1: tensor(121.6192, device='cuda:0') (tensor(132.2903, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 1.0291 (0.8892723916615642) + | > loader_time: 0.02 (0.013836215365821023) + + + --> TIME: 2024-06-25 03:16:06 -- STEP: 468/591 -- GLOBAL_STEP: 1650 + | > loss_disc: 2.6149346828460693 (2.549612174686205) + | > loss_disc_real_0: 0.3317308723926544 (0.21629106732578868) + | > loss_disc_real_1: 0.24629388749599457 (0.21352689509463108) + | > loss_disc_real_2: 0.2122294008731842 (0.2273721137267148) + | > loss_disc_real_3: 0.21201428771018982 (0.2371057190446772) + | > loss_disc_real_4: 0.24160821735858917 (0.23299572408262992) + | > loss_disc_real_5: 0.23120862245559692 (0.22567265090715682) + | > loss_0: 2.6149346828460693 (2.549612174686205) + | > grad_norm_0: tensor(15.7952, device='cuda:0') (tensor(9.2084, device='cuda:0')) + | > loss_gen: 2.093061923980713 (2.1441418928468368) + | > loss_kl: 2.295858860015869 (2.1932829182881584) + | > loss_feat: 2.88287615776062 (3.246852175292806) + | > loss_mel: 30.819711685180664 (31.62174603470372) + | > loss_duration: 1.6651297807693481 (1.6048043744686327) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 39.75664138793945 (40.81082742642129) + | > grad_norm_1: tensor(141.9292, device='cuda:0') (tensor(134.0832, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 1.0107 (0.8961319745096384) + | > loader_time: 0.018 (0.014139518778548276) + + + --> TIME: 2024-06-25 03:16:33 -- STEP: 493/591 -- GLOBAL_STEP: 1675 + | > loss_disc: 2.455625057220459 (2.549628413473134) + | > loss_disc_real_0: 0.2080194354057312 (0.21577447618057471) + | > loss_disc_real_1: 0.15864287316799164 (0.21329765599107645) + | > loss_disc_real_2: 0.22028766572475433 (0.2271103000090766) + | > loss_disc_real_3: 0.1648198515176773 (0.23715551635798285) + | > loss_disc_real_4: 0.19353118538856506 (0.2329264494909718) + | > loss_disc_real_5: 0.24084177613258362 (0.22578753357182652) + | > loss_0: 2.455625057220459 (2.549628413473134) + | > grad_norm_0: tensor(5.6167, device='cuda:0') (tensor(9.1765, device='cuda:0')) + | > loss_gen: 2.045146942138672 (2.1402441848364164) + | > loss_kl: 2.308143377304077 (2.1947617949143394) + | > loss_feat: 3.151275634765625 (3.2384448259403937) + | > loss_mel: 30.455711364746094 (31.556211011163136) + | > loss_duration: 1.6321485042572021 (1.607586753295837) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 39.59242630004883 (40.73724857353531) + | > grad_norm_1: tensor(104.5618, device='cuda:0') (tensor(133.8996, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 1.0203 (0.9038605506231527) + | > loader_time: 0.0185 (0.014421617283540609) + + + --> TIME: 2024-06-25 03:17:01 -- STEP: 518/591 -- GLOBAL_STEP: 1700 + | > loss_disc: 2.4280896186828613 (2.5486499485362) + | > loss_disc_real_0: 0.30557936429977417 (0.2156517886373297) + | > loss_disc_real_1: 0.19423922896385193 (0.21326076837695243) + | > loss_disc_real_2: 0.18261267244815826 (0.22691976487579038) + | > loss_disc_real_3: 0.23361274600028992 (0.23689908813325594) + | > loss_disc_real_4: 0.21810998022556305 (0.2327457373695714) + | > loss_disc_real_5: 0.21628369390964508 (0.22585512448864553) + | > loss_0: 2.4280896186828613 (2.5486499485362) + | > grad_norm_0: tensor(15.2376, device='cuda:0') (tensor(9.3323, device='cuda:0')) + | > loss_gen: 1.9111056327819824 (2.138110892192741) + | > loss_kl: 2.2662241458892822 (2.196588756725135) + | > loss_feat: 3.357872486114502 (3.2348595484803546) + | > loss_mel: 30.615577697753906 (31.481969770777653) + | > loss_duration: 1.6433029174804688 (1.6104842475022132) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 39.79408264160156 (40.66201322695459) + | > grad_norm_1: tensor(90.9078, device='cuda:0') (tensor(132.9521, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 1.104 (0.9126283298588166) + | > loader_time: 0.0209 (0.014703263647307755) + + + --> TIME: 2024-06-25 03:17:30 -- STEP: 543/591 -- GLOBAL_STEP: 1725 + | > loss_disc: 2.5686326026916504 (2.549502555836631) + | > loss_disc_real_0: 0.23018042743206024 (0.21538877614789245) + | > loss_disc_real_1: 0.1484200656414032 (0.21312636540410268) + | > loss_disc_real_2: 0.16364786028862 (0.22674915097091305) + | > loss_disc_real_3: 0.17961640655994415 (0.23671351880877936) + | > loss_disc_real_4: 0.2115764170885086 (0.23264193827111415) + | > loss_disc_real_5: 0.26545801758766174 (0.22617903166548323) + | > loss_0: 2.5686326026916504 (2.549502555836631) + | > grad_norm_0: tensor(9.3636, device='cuda:0') (tensor(9.3816, device='cuda:0')) + | > loss_gen: 1.8966368436813354 (2.134373877369022) + | > loss_kl: 2.328063726425171 (2.1974259153914066) + | > loss_feat: 2.8808541297912598 (3.2262307271553445) + | > loss_mel: 29.645627975463867 (31.402803670416013) + | > loss_duration: 1.687569260597229 (1.6134049416027463) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 38.438751220703125 (40.57423915125385) + | > grad_norm_1: tensor(242.1570, device='cuda:0') (tensor(133.3172, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 1.2195 (0.9226378200242972) + | > loader_time: 0.0239 (0.015037406654428155) + + + --> TIME: 2024-06-25 03:18:00 -- STEP: 568/591 -- GLOBAL_STEP: 1750 + | > loss_disc: 2.571948289871216 (2.5513658515164557) + | > loss_disc_real_0: 0.2162306308746338 (0.21555924896990336) + | > loss_disc_real_1: 0.19702453911304474 (0.2133058173660661) + | > loss_disc_real_2: 0.20379456877708435 (0.22705416209523538) + | > loss_disc_real_3: 0.24135388433933258 (0.2363886176018228) + | > loss_disc_real_4: 0.1589856892824173 (0.232525898078063) + | > loss_disc_real_5: 0.19869668781757355 (0.22606406522206438) + | > loss_0: 2.571948289871216 (2.5513658515164557) + | > grad_norm_0: tensor(6.6850, device='cuda:0') (tensor(9.5597, device='cuda:0')) + | > loss_gen: 2.274778366088867 (2.1321879106927963) + | > loss_kl: 2.250199317932129 (2.199205410102724) + | > loss_feat: 3.1069583892822266 (3.217205493802755) + | > loss_mel: 29.60345458984375 (31.340425393950785) + | > loss_duration: 1.6700451374053955 (1.6162808361187795) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 38.905433654785156 (40.50530505516157) + | > grad_norm_1: tensor(141.6699, device='cuda:0') (tensor(133.6657, device='cuda:0')) + | > current_lr_0: 0.000199950003125 + | > current_lr_1: 0.000199950003125 + | > step_time: 1.1497 (0.9336478626224357) + | > loader_time: 0.022 (0.015397113393729836) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003855258860486619 (+0.00010104382291753217) + | > avg_loss_disc: 2.6014753757639144 (+0.02384423702321259) + | > avg_loss_disc_real_0: 0.25269244199103497 (+0.07631685029953084) + | > avg_loss_disc_real_1: 0.18640634417533875 (-0.080355186094629) + | > avg_loss_disc_real_2: 0.18898523170897302 (-0.01572046682555625) + | > avg_loss_disc_real_3: 0.20032338211511044 (-0.022658841882614383) + | > avg_loss_disc_real_4: 0.2426393802495713 (+0.022020696325505046) + | > avg_loss_disc_real_5: 0.18284591747091172 (+0.0006900199867309609) + | > avg_loss_0: 2.6014753757639144 (+0.02384423702321259) + | > avg_loss_gen: 1.8999644898353738 (-0.15373877768820954) + | > avg_loss_kl: 2.3112471839214894 (+0.011625452244534706) + | > avg_loss_feat: 3.1885354671072457 (-0.27684007806980837) + | > avg_loss_mel: 30.537570222895194 (-2.515191747787153) + | > avg_loss_duration: 1.628951810775919 (-0.02060557172653521) + | > avg_loss_1: 39.566269245553535 (-2.9547504668540014) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_1773.pth + + > EPOCH: 4/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 03:18:41)  + + --> TIME: 2024-06-25 03:18:45 -- STEP: 2/591 -- GLOBAL_STEP: 1775 + | > loss_disc: 2.7233548164367676 (2.6922322511672974) + | > loss_disc_real_0: 0.13759931921958923 (0.2212103009223938) + | > loss_disc_real_1: 0.21448764204978943 (0.29548826813697815) + | > loss_disc_real_2: 0.21251076459884644 (0.26747047901153564) + | > loss_disc_real_3: 0.2499895840883255 (0.26801495999097824) + | > loss_disc_real_4: 0.2523762881755829 (0.23370475322008133) + | > loss_disc_real_5: 0.32130569219589233 (0.26168716698884964) + | > loss_0: 2.7233548164367676 (2.6922322511672974) + | > grad_norm_0: tensor(18.1469, device='cuda:0') (tensor(16.9164, device='cuda:0')) + | > loss_gen: 2.0599303245544434 (2.1185362339019775) + | > loss_kl: 2.753063678741455 (2.66823410987854) + | > loss_feat: 3.517859935760498 (3.433039903640747) + | > loss_mel: 32.08041763305664 (31.39147663116455) + | > loss_duration: 1.5399723052978516 (1.5564444065093994) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 41.95124053955078 (41.1677303314209) + | > grad_norm_1: tensor(225.7782, device='cuda:0') (tensor(162.1790, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.767 (0.771065354347229) + | > loader_time: 0.0076 (0.008115053176879883) + + + --> TIME: 2024-06-25 03:19:05 -- STEP: 27/591 -- GLOBAL_STEP: 1800 + | > loss_disc: 2.5038485527038574 (2.6112635135650635) + | > loss_disc_real_0: 0.1691834032535553 (0.22355562724449016) + | > loss_disc_real_1: 0.1799367368221283 (0.22167901970722056) + | > loss_disc_real_2: 0.16353946924209595 (0.23024545499572047) + | > loss_disc_real_3: 0.2473972886800766 (0.23982432705384713) + | > loss_disc_real_4: 0.27285856008529663 (0.23840942868479975) + | > loss_disc_real_5: 0.2612406015396118 (0.23750213892371566) + | > loss_0: 2.5038485527038574 (2.6112635135650635) + | > grad_norm_0: tensor(9.6436, device='cuda:0') (tensor(10.4802, device='cuda:0')) + | > loss_gen: 1.9423332214355469 (2.1032943284070047) + | > loss_kl: 2.5471107959747314 (2.496168163087633) + | > loss_feat: 3.320070505142212 (3.2679627118287264) + | > loss_mel: 29.8272647857666 (30.140645698264795) + | > loss_duration: 1.561274528503418 (1.5466195698137637) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 39.198055267333984 (39.55468990184644) + | > grad_norm_1: tensor(141.7817, device='cuda:0') (tensor(132.3935, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.7959 (0.7836026438960323) + | > loader_time: 0.0106 (0.009269016760366936) + + + --> TIME: 2024-06-25 03:19:26 -- STEP: 52/591 -- GLOBAL_STEP: 1825 + | > loss_disc: 2.6504154205322266 (2.6033699695880608) + | > loss_disc_real_0: 0.19675177335739136 (0.22000039225587478) + | > loss_disc_real_1: 0.258228063583374 (0.2188417109159323) + | > loss_disc_real_2: 0.2554422616958618 (0.22875696162764841) + | > loss_disc_real_3: 0.17432326078414917 (0.23656463651702955) + | > loss_disc_real_4: 0.25186365842819214 (0.2348710335791111) + | > loss_disc_real_5: 0.25123757123947144 (0.23341710951465827) + | > loss_0: 2.6504154205322266 (2.6033699695880608) + | > grad_norm_0: tensor(7.0532, device='cuda:0') (tensor(10.1588, device='cuda:0')) + | > loss_gen: 2.254042148590088 (2.0931586554417256) + | > loss_kl: 2.398508071899414 (2.442946255207062) + | > loss_feat: 3.067112684249878 (3.2396131112025333) + | > loss_mel: 29.94823455810547 (30.183713362767147) + | > loss_duration: 1.5871198177337646 (1.574864807037207) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 39.25502014160156 (39.53429559560922) + | > grad_norm_1: tensor(235.2081, device='cuda:0') (tensor(143.9648, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.8171 (0.7940678734045762) + | > loader_time: 0.0115 (0.009808246905987084) + + + --> TIME: 2024-06-25 03:19:47 -- STEP: 77/591 -- GLOBAL_STEP: 1850 + | > loss_disc: 2.5477287769317627 (2.5975603283225728) + | > loss_disc_real_0: 0.2147529572248459 (0.22014489557061875) + | > loss_disc_real_1: 0.23538726568222046 (0.21689146350730548) + | > loss_disc_real_2: 0.23536261916160583 (0.22687855924104716) + | > loss_disc_real_3: 0.20931880176067352 (0.23750204215576123) + | > loss_disc_real_4: 0.20660851895809174 (0.23421285465940253) + | > loss_disc_real_5: 0.20085930824279785 (0.2320966198072805) + | > loss_0: 2.5477287769317627 (2.5975603283225728) + | > grad_norm_0: tensor(5.7574, device='cuda:0') (tensor(10.6474, device='cuda:0')) + | > loss_gen: 1.9939218759536743 (2.077962669459257) + | > loss_kl: 2.275522470474243 (2.415394337146313) + | > loss_feat: 3.2342162132263184 (3.206291610544378) + | > loss_mel: 29.28583526611328 (29.990788348309405) + | > loss_duration: 1.6619335412979126 (1.5880965722071663) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 38.45143127441406 (39.27853309334099) + | > grad_norm_1: tensor(76.2155, device='cuda:0') (tensor(135.5883, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.828 (0.8040050995814336) + | > loader_time: 0.0106 (0.010103284538566293) + + + --> TIME: 2024-06-25 03:20:08 -- STEP: 102/591 -- GLOBAL_STEP: 1875 + | > loss_disc: 2.5902411937713623 (2.6073709118599995) + | > loss_disc_real_0: 0.2569839358329773 (0.22049385900883114) + | > loss_disc_real_1: 0.21816103160381317 (0.21854373783457512) + | > loss_disc_real_2: 0.18550987541675568 (0.2264966285404037) + | > loss_disc_real_3: 0.19378702342510223 (0.2368713916808951) + | > loss_disc_real_4: 0.2217164784669876 (0.2348313584339385) + | > loss_disc_real_5: 0.19645503163337708 (0.23253304262955984) + | > loss_0: 2.5902411937713623 (2.6073709118599995) + | > grad_norm_0: tensor(8.2328, device='cuda:0') (tensor(10.7217, device='cuda:0')) + | > loss_gen: 2.0359115600585938 (2.0698567547050186) + | > loss_kl: 2.402111768722534 (2.405743601275424) + | > loss_feat: 3.1825931072235107 (3.1718686935948393) + | > loss_mel: 30.173885345458984 (29.961200770209818) + | > loss_duration: 1.6524455547332764 (1.5996868236392152) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 39.44694900512695 (39.208356520708854) + | > grad_norm_1: tensor(119.1027, device='cuda:0') (tensor(140.3973, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.8401 (0.8109362452637916) + | > loader_time: 0.0125 (0.01046253653133617) + + + --> TIME: 2024-06-25 03:20:29 -- STEP: 127/591 -- GLOBAL_STEP: 1900 + | > loss_disc: 2.67318058013916 (2.6065977858746154) + | > loss_disc_real_0: 0.24406574666500092 (0.21953353890049176) + | > loss_disc_real_1: 0.2338249385356903 (0.21874861360534908) + | > loss_disc_real_2: 0.20981146395206451 (0.22754039961522018) + | > loss_disc_real_3: 0.2657395303249359 (0.23692503219514383) + | > loss_disc_real_4: 0.2578112483024597 (0.23431982745335797) + | > loss_disc_real_5: 0.259528785943985 (0.23208404831060275) + | > loss_0: 2.67318058013916 (2.6065977858746154) + | > grad_norm_0: tensor(6.0982, device='cuda:0') (tensor(10.6017, device='cuda:0')) + | > loss_gen: 1.8804818391799927 (2.062445130873853) + | > loss_kl: 2.465733051300049 (2.4023050961531984) + | > loss_feat: 3.0360348224639893 (3.14850749744205) + | > loss_mel: 30.067649841308594 (29.907975294458584) + | > loss_duration: 1.6101449728012085 (1.603778224291764) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 39.0600471496582 (39.12501129390685) + | > grad_norm_1: tensor(176.5140, device='cuda:0') (tensor(142.4745, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.8385 (0.8164118650391345) + | > loader_time: 0.0119 (0.010682558450173208) + + + --> TIME: 2024-06-25 03:20:51 -- STEP: 152/591 -- GLOBAL_STEP: 1925 + | > loss_disc: 2.6377627849578857 (2.6101924557434892) + | > loss_disc_real_0: 0.27585750818252563 (0.22088254870552765) + | > loss_disc_real_1: 0.2292032390832901 (0.21815241883067713) + | > loss_disc_real_2: 0.30321016907691956 (0.2271858927254614) + | > loss_disc_real_3: 0.31025099754333496 (0.23670321712760548) + | > loss_disc_real_4: 0.2269805520772934 (0.2336750488336149) + | > loss_disc_real_5: 0.2563323676586151 (0.2321990955817072) + | > loss_0: 2.6377627849578857 (2.6101924557434892) + | > grad_norm_0: tensor(11.3844, device='cuda:0') (tensor(11.1141, device='cuda:0')) + | > loss_gen: 1.9703431129455566 (2.0586422744550195) + | > loss_kl: 2.4406425952911377 (2.3968058087323825) + | > loss_feat: 3.0407907962799072 (3.1369965421526054) + | > loss_mel: 29.857946395874023 (29.838390400535182) + | > loss_duration: 1.6203879117965698 (1.604455222424708) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 38.930110931396484 (39.035290366724915) + | > grad_norm_1: tensor(161.3742, device='cuda:0') (tensor(143.3845, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.8577 (0.8216922157689146) + | > loader_time: 0.0116 (0.010894962047275743) + + + --> TIME: 2024-06-25 03:21:13 -- STEP: 177/591 -- GLOBAL_STEP: 1950 + | > loss_disc: 2.6687307357788086 (2.61646285030128) + | > loss_disc_real_0: 0.2915922999382019 (0.22025039047002792) + | > loss_disc_real_1: 0.1355089545249939 (0.21875822720891341) + | > loss_disc_real_2: 0.2295498102903366 (0.22800554242511253) + | > loss_disc_real_3: 0.21887069940567017 (0.23647075723120048) + | > loss_disc_real_4: 0.20007561147212982 (0.23403517388354586) + | > loss_disc_real_5: 0.21720105409622192 (0.2323521900143327) + | > loss_0: 2.6687307357788086 (2.61646285030128) + | > grad_norm_0: tensor(13.9211, device='cuda:0') (tensor(11.0830, device='cuda:0')) + | > loss_gen: 2.1975817680358887 (2.052851950381434) + | > loss_kl: 2.298975706100464 (2.3905724522757663) + | > loss_feat: 3.0098776817321777 (3.1133103357196528) + | > loss_mel: 28.89458656311035 (29.74111382435944) + | > loss_duration: 1.5974829196929932 (1.6029879710095076) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 37.998504638671875 (38.90083662130065) + | > grad_norm_1: tensor(39.6826, device='cuda:0') (tensor(138.3771, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.8692 (0.8272007750926045) + | > loader_time: 0.0121 (0.011078791429767504) + + + --> TIME: 2024-06-25 03:21:35 -- STEP: 202/591 -- GLOBAL_STEP: 1975 + | > loss_disc: 2.572537660598755 (2.6215959079194784) + | > loss_disc_real_0: 0.28325238823890686 (0.2201686628753006) + | > loss_disc_real_1: 0.16162945330142975 (0.21933842472510767) + | > loss_disc_real_2: 0.20992209017276764 (0.22821226236548756) + | > loss_disc_real_3: 0.27647385001182556 (0.23704499426749673) + | > loss_disc_real_4: 0.21021385490894318 (0.23490966598291207) + | > loss_disc_real_5: 0.24579092860221863 (0.2324749139128345) + | > loss_0: 2.572537660598755 (2.6215959079194784) + | > grad_norm_0: tensor(13.0280, device='cuda:0') (tensor(11.2663, device='cuda:0')) + | > loss_gen: 1.8403751850128174 (2.046005636158555) + | > loss_kl: 2.279482126235962 (2.380263871485644) + | > loss_feat: 3.032444477081299 (3.0909600305085134) + | > loss_mel: 29.574237823486328 (29.717040581278283) + | > loss_duration: 1.5724339485168457 (1.6028964200822433) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 38.298973083496094 (38.837166606789765) + | > grad_norm_1: tensor(154.5450, device='cuda:0') (tensor(135.8121, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.8876 (0.832266107644185) + | > loader_time: 0.0135 (0.011239171028137209) + + + --> TIME: 2024-06-25 03:21:57 -- STEP: 227/591 -- GLOBAL_STEP: 2000 + | > loss_disc: 2.4990808963775635 (2.627122670018201) + | > loss_disc_real_0: 0.20355993509292603 (0.21962251888366524) + | > loss_disc_real_1: 0.18462450802326202 (0.21997728234870847) + | > loss_disc_real_2: 0.19143269956111908 (0.2284522259419185) + | > loss_disc_real_3: 0.20465001463890076 (0.23669544945466886) + | > loss_disc_real_4: 0.16922971606254578 (0.23471607563254065) + | > loss_disc_real_5: 0.16298191249370575 (0.23302198093105517) + | > loss_0: 2.4990808963775635 (2.627122670018201) + | > grad_norm_0: tensor(5.4785, device='cuda:0') (tensor(11.1833, device='cuda:0')) + | > loss_gen: 2.176172971725464 (2.038665392850462) + | > loss_kl: 2.341538190841675 (2.3778980460986157) + | > loss_feat: 3.271287679672241 (3.0630393017756257) + | > loss_mel: 29.315927505493164 (29.665648078078217) + | > loss_duration: 1.6124050617218018 (1.6031578553405628) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 38.71733093261719 (38.748408683071034) + | > grad_norm_1: tensor(87.9230, device='cuda:0') (tensor(134.1518, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.8888 (0.837145522827619) + | > loader_time: 0.0136 (0.011463444663564542) + + + --> TIME: 2024-06-25 03:22:20 -- STEP: 252/591 -- GLOBAL_STEP: 2025 + | > loss_disc: 2.600785255432129 (2.632358337205554) + | > loss_disc_real_0: 0.2871609330177307 (0.22044151492180333) + | > loss_disc_real_1: 0.2526951730251312 (0.22039489176065205) + | > loss_disc_real_2: 0.22810006141662598 (0.2290461307007169) + | > loss_disc_real_3: 0.2259732335805893 (0.2368023701248661) + | > loss_disc_real_4: 0.27390962839126587 (0.2351794314999429) + | > loss_disc_real_5: 0.2283262014389038 (0.23310956607262293) + | > loss_0: 2.600785255432129 (2.632358337205554) + | > grad_norm_0: tensor(11.4386, device='cuda:0') (tensor(11.2972, device='cuda:0')) + | > loss_gen: 2.007768154144287 (2.031910641798895) + | > loss_kl: 2.3931169509887695 (2.3754560285144373) + | > loss_feat: 2.7012903690338135 (3.037600845571548) + | > loss_mel: 28.301862716674805 (29.594528546409002) + | > loss_duration: 1.590999722480774 (1.6040063784236) + | > amp_scaler: 1024.0 (552.6349206349208) + | > loss_1: 36.99503707885742 (38.643502432202524) + | > grad_norm_1: tensor(72.9482, device='cuda:0') (tensor(133.1778, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.8878 (0.8423789190867591) + | > loader_time: 0.0155 (0.011736090221102276) + + + --> TIME: 2024-06-25 03:22:43 -- STEP: 277/591 -- GLOBAL_STEP: 2050 + | > loss_disc: 2.7387187480926514 (2.630551844297334) + | > loss_disc_real_0: 0.2257586419582367 (0.21920987256274757) + | > loss_disc_real_1: 0.22962704300880432 (0.22033932127246794) + | > loss_disc_real_2: 0.29987069964408875 (0.229133209166544) + | > loss_disc_real_3: 0.2687373161315918 (0.23669944106456606) + | > loss_disc_real_4: 0.259154349565506 (0.234954211417088) + | > loss_disc_real_5: 0.2617434561252594 (0.2330663184289037) + | > loss_0: 2.7387187480926514 (2.630551844297334) + | > grad_norm_0: tensor(6.4321, device='cuda:0') (tensor(10.8922, device='cuda:0')) + | > loss_gen: 1.810405969619751 (2.0283564815452353) + | > loss_kl: 2.2476232051849365 (2.3730016658452437) + | > loss_feat: 2.8397977352142334 (3.029669922611774) + | > loss_mel: 28.922536849975586 (29.533271782665047) + | > loss_duration: 1.6060765981674194 (1.6045987455423127) + | > amp_scaler: 1024.0 (595.1768953068594) + | > loss_1: 37.42643737792969 (38.56889855560411) + | > grad_norm_1: tensor(151.2142, device='cuda:0') (tensor(133.2510, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.9004 (0.8473130279499701) + | > loader_time: 0.0132 (0.012021903957270546) + + + --> TIME: 2024-06-25 03:23:06 -- STEP: 302/591 -- GLOBAL_STEP: 2075 + | > loss_disc: 2.636394739151001 (2.634671730710971) + | > loss_disc_real_0: 0.27897515892982483 (0.21970201199812606) + | > loss_disc_real_1: 0.19876472651958466 (0.22115401484516292) + | > loss_disc_real_2: 0.2684052884578705 (0.22939495905148272) + | > loss_disc_real_3: 0.18561594188213348 (0.23632407178547207) + | > loss_disc_real_4: 0.22036631405353546 (0.23515311026612654) + | > loss_disc_real_5: 0.1791953593492508 (0.23317827111640513) + | > loss_0: 2.636394739151001 (2.634671730710971) + | > grad_norm_0: tensor(11.9741, device='cuda:0') (tensor(11.2837, device='cuda:0')) + | > loss_gen: 1.8048804998397827 (2.024979447686906) + | > loss_kl: 2.166773796081543 (2.3719169027757956) + | > loss_feat: 2.760636329650879 (3.016612728699943) + | > loss_mel: 29.542234420776367 (29.482148549414628) + | > loss_duration: 1.5717816352844238 (1.6054511666297908) + | > amp_scaler: 1024.0 (630.6754966887419) + | > loss_1: 37.84630584716797 (38.50110877586516) + | > grad_norm_1: tensor(150.4440, device='cuda:0') (tensor(131.4658, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.944 (0.8525823449456928) + | > loader_time: 0.0165 (0.012314337768302062) + + + --> TIME: 2024-06-25 03:23:30 -- STEP: 327/591 -- GLOBAL_STEP: 2100 + | > loss_disc: 2.701413154602051 (2.635801627366185) + | > loss_disc_real_0: 0.2358604371547699 (0.21949552596526772) + | > loss_disc_real_1: 0.21793848276138306 (0.22117021889497024) + | > loss_disc_real_2: 0.1859017014503479 (0.22956149817789) + | > loss_disc_real_3: 0.21910293400287628 (0.23647378553673398) + | > loss_disc_real_4: 0.2655009329319 (0.23529858340363985) + | > loss_disc_real_5: 0.25962138175964355 (0.23368542074063503) + | > loss_0: 2.701413154602051 (2.635801627366185) + | > grad_norm_0: tensor(5.0344, device='cuda:0') (tensor(11.2003, device='cuda:0')) + | > loss_gen: 1.6458191871643066 (2.021706347071794) + | > loss_kl: 2.3219051361083984 (2.369135296672855) + | > loss_feat: 2.569946765899658 (3.0057876117367988) + | > loss_mel: 28.616287231445312 (29.42666046189241) + | > loss_duration: 1.6189333200454712 (1.6065882663114355) + | > amp_scaler: 512.0 (656.0489296636091) + | > loss_1: 36.77289581298828 (38.429878013214186) + | > grad_norm_1: tensor(184.9049, device='cuda:0') (tensor(129.7139, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.9351 (0.8576216056077123) + | > loader_time: 0.0158 (0.012602119270814668) + + + --> TIME: 2024-06-25 03:23:54 -- STEP: 352/591 -- GLOBAL_STEP: 2125 + | > loss_disc: 2.7152788639068604 (2.635991746051744) + | > loss_disc_real_0: 0.2759435176849365 (0.21942616285840896) + | > loss_disc_real_1: 0.25095996260643005 (0.22140655835921122) + | > loss_disc_real_2: 0.21299874782562256 (0.2294412012296644) + | > loss_disc_real_3: 0.19514715671539307 (0.2358927110802721) + | > loss_disc_real_4: 0.21997876465320587 (0.2353524531830441) + | > loss_disc_real_5: 0.25585347414016724 (0.23353287658061495) + | > loss_0: 2.7152788639068604 (2.635991746051744) + | > grad_norm_0: tensor(10.7255, device='cuda:0') (tensor(11.3511, device='cuda:0')) + | > loss_gen: 2.0763819217681885 (2.0217660221863856) + | > loss_kl: 2.4399940967559814 (2.366792586716737) + | > loss_feat: 2.8736732006073 (2.997972124002196) + | > loss_mel: 29.532608032226562 (29.392586978999052) + | > loss_duration: 1.6277048587799072 (1.6077011688189073) + | > amp_scaler: 512.0 (645.8181818181824) + | > loss_1: 38.55036163330078 (38.38681890747764) + | > grad_norm_1: tensor(150.2968, device='cuda:0') (tensor(131.4099, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.9362 (0.8629312732002952) + | > loader_time: 0.0186 (0.012884231453592127) + + + --> TIME: 2024-06-25 03:24:18 -- STEP: 377/591 -- GLOBAL_STEP: 2150 + | > loss_disc: 2.8125975131988525 (2.637797313912794) + | > loss_disc_real_0: 0.2694644331932068 (0.2192413505532381) + | > loss_disc_real_1: 0.21959099173545837 (0.22155085482711193) + | > loss_disc_real_2: 0.1776241660118103 (0.2298203175912169) + | > loss_disc_real_3: 0.23798488080501556 (0.23584841962518363) + | > loss_disc_real_4: 0.3230445683002472 (0.23545552458149685) + | > loss_disc_real_5: 0.2736799716949463 (0.23366308168802083) + | > loss_0: 2.8125975131988525 (2.637797313912794) + | > grad_norm_0: tensor(9.3247, device='cuda:0') (tensor(11.2960, device='cuda:0')) + | > loss_gen: 1.776278018951416 (2.0188092426532753) + | > loss_kl: 2.3572635650634766 (2.36361917956122) + | > loss_feat: 2.4379539489746094 (2.9852835632445633) + | > loss_mel: 27.640058517456055 (29.332715522705403) + | > loss_duration: 1.6360530853271484 (1.6092469138871452) + | > amp_scaler: 512.0 (636.9442970822288) + | > loss_1: 35.84761047363281 (38.3096744527235) + | > grad_norm_1: tensor(148.2894, device='cuda:0') (tensor(129.7255, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 1.0074 (0.8684891136634887) + | > loader_time: 0.0174 (0.013126337876054273) + + + --> TIME: 2024-06-25 03:24:42 -- STEP: 402/591 -- GLOBAL_STEP: 2175 + | > loss_disc: 2.6577398777008057 (2.639377649743758) + | > loss_disc_real_0: 0.2658148407936096 (0.21908141485420032) + | > loss_disc_real_1: 0.21371009945869446 (0.22173557626963855) + | > loss_disc_real_2: 0.2547677159309387 (0.23004948080920462) + | > loss_disc_real_3: 0.2519756555557251 (0.2357380120315362) + | > loss_disc_real_4: 0.24079100787639618 (0.23529017067963803) + | > loss_disc_real_5: 0.2818790376186371 (0.23349754653166777) + | > loss_0: 2.6577398777008057 (2.639377649743758) + | > grad_norm_0: tensor(8.6794, device='cuda:0') (tensor(11.2043, device='cuda:0')) + | > loss_gen: 2.114914655685425 (2.0153647444734513) + | > loss_kl: 2.225158452987671 (2.3635154915093186) + | > loss_feat: 2.7438580989837646 (2.970676184293642) + | > loss_mel: 29.149784088134766 (29.28717275401253) + | > loss_duration: 1.6727761030197144 (1.6101975473598462) + | > amp_scaler: 512.0 (629.1741293532343) + | > loss_1: 37.906490325927734 (38.24692672046262) + | > grad_norm_1: tensor(167.6822, device='cuda:0') (tensor(130.8815, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.9644 (0.8744566861669816) + | > loader_time: 0.0181 (0.013412448304209543) + + + --> TIME: 2024-06-25 03:25:08 -- STEP: 427/591 -- GLOBAL_STEP: 2200 + | > loss_disc: 2.661332130432129 (2.6405636828732986) + | > loss_disc_real_0: 0.2690832316875458 (0.21936853488239808) + | > loss_disc_real_1: 0.22024065256118774 (0.22170118464100283) + | > loss_disc_real_2: 0.2747499644756317 (0.23014785185351583) + | > loss_disc_real_3: 0.23715968430042267 (0.2355873992348164) + | > loss_disc_real_4: 0.26638421416282654 (0.2354475805575172) + | > loss_disc_real_5: 0.20951728522777557 (0.23376211764801302) + | > loss_0: 2.661332130432129 (2.6405636828732986) + | > grad_norm_0: tensor(8.9872, device='cuda:0') (tensor(11.2081, device='cuda:0')) + | > loss_gen: 1.7976421117782593 (2.0130699470953277) + | > loss_kl: 2.4253575801849365 (2.3631778159800394) + | > loss_feat: 2.961263418197632 (2.9622607342811595) + | > loss_mel: 28.85650062561035 (29.222418626521733) + | > loss_duration: 1.608973503112793 (1.6106256229816052) + | > amp_scaler: 512.0 (622.3138173302114) + | > loss_1: 37.64973831176758 (38.17155275188509) + | > grad_norm_1: tensor(169.0814, device='cuda:0') (tensor(130.4237, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.9813 (0.8809098376602423) + | > loader_time: 0.0169 (0.013681006934101184) + + + --> TIME: 2024-06-25 03:25:33 -- STEP: 452/591 -- GLOBAL_STEP: 2225 + | > loss_disc: 2.749490737915039 (2.642873228651232) + | > loss_disc_real_0: 0.385308176279068 (0.21940627849075647) + | > loss_disc_real_1: 0.1990308314561844 (0.22204479083946327) + | > loss_disc_real_2: 0.1975516378879547 (0.23044274381436078) + | > loss_disc_real_3: 0.22006726264953613 (0.2357988856666911) + | > loss_disc_real_4: 0.2713507115840912 (0.23553615881014714) + | > loss_disc_real_5: 0.2836378514766693 (0.23381957777701648) + | > loss_0: 2.749490737915039 (2.642873228651232) + | > grad_norm_0: tensor(23.7810, device='cuda:0') (tensor(11.1963, device='cuda:0')) + | > loss_gen: 1.6452157497406006 (2.010978089233413) + | > loss_kl: 2.2715210914611816 (2.359861373901367) + | > loss_feat: 2.573591947555542 (2.9520982920596026) + | > loss_mel: 27.839542388916016 (29.173510728684146) + | > loss_duration: 1.604630947113037 (1.611318065529376) + | > amp_scaler: 512.0 (616.2123893805315) + | > loss_1: 35.93449783325195 (38.107766581847606) + | > grad_norm_1: tensor(116.3336, device='cuda:0') (tensor(130.4909, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 0.9958 (0.8880748827900506) + | > loader_time: 0.0223 (0.013998993730123064) + + + --> TIME: 2024-06-25 03:26:00 -- STEP: 477/591 -- GLOBAL_STEP: 2250 + | > loss_disc: 2.6681506633758545 (2.6445254664750966) + | > loss_disc_real_0: 0.24119824171066284 (0.21935036387958368) + | > loss_disc_real_1: 0.2277887463569641 (0.22228366883170936) + | > loss_disc_real_2: 0.227451890707016 (0.23051430247489763) + | > loss_disc_real_3: 0.21477879583835602 (0.23574422877789544) + | > loss_disc_real_4: 0.19472260773181915 (0.2353816590031738) + | > loss_disc_real_5: 0.20158839225769043 (0.23396708983420325) + | > loss_0: 2.6681506633758545 (2.6445254664750966) + | > grad_norm_0: tensor(7.5327, device='cuda:0') (tensor(11.2054, device='cuda:0')) + | > loss_gen: 1.8602497577667236 (2.008152261220181) + | > loss_kl: 2.261709451675415 (2.3575061882067025) + | > loss_feat: 2.6218926906585693 (2.9407520154237243) + | > loss_mel: 28.655969619750977 (29.129161282905244) + | > loss_duration: 1.625281572341919 (1.6121580745689024) + | > amp_scaler: 512.0 (610.7505241090151) + | > loss_1: 37.02510452270508 (38.0477298520646) + | > grad_norm_1: tensor(96.8906, device='cuda:0') (tensor(129.9104, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 1.0176 (0.895317750406965) + | > loader_time: 0.0159 (0.014293118342913422) + + + --> TIME: 2024-06-25 03:26:27 -- STEP: 502/591 -- GLOBAL_STEP: 2275 + | > loss_disc: 2.6323535442352295 (2.644010639285661) + | > loss_disc_real_0: 0.26442593336105347 (0.21919596863101679) + | > loss_disc_real_1: 0.2280873954296112 (0.2221947029767284) + | > loss_disc_real_2: 0.18949581682682037 (0.230606108607049) + | > loss_disc_real_3: 0.2721056342124939 (0.23565443435988104) + | > loss_disc_real_4: 0.20019397139549255 (0.23527398678767253) + | > loss_disc_real_5: 0.21640099585056305 (0.23390713170823824) + | > loss_0: 2.6323535442352295 (2.644010639285661) + | > grad_norm_0: tensor(10.0141, device='cuda:0') (tensor(11.2341, device='cuda:0')) + | > loss_gen: 1.7535552978515625 (2.0066635511310884) + | > loss_kl: 2.2422144412994385 (2.3552843104320695) + | > loss_feat: 2.57881236076355 (2.9349737680290797) + | > loss_mel: 27.29275131225586 (29.06657866747731) + | > loss_duration: 1.6496515274047852 (1.6128361901913981) + | > amp_scaler: 512.0 (605.8326693227096) + | > loss_1: 35.51698303222656 (37.97633649058551) + | > grad_norm_1: tensor(71.6567, device='cuda:0') (tensor(129.8865, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 1.082 (0.903941696858501) + | > loader_time: 0.0244 (0.014576969868633377) + + + --> TIME: 2024-06-25 03:26:55 -- STEP: 527/591 -- GLOBAL_STEP: 2300 + | > loss_disc: 2.583225965499878 (2.6443063095353354) + | > loss_disc_real_0: 0.1613394170999527 (0.21892655377878864) + | > loss_disc_real_1: 0.20436792075634003 (0.22226948193513016) + | > loss_disc_real_2: 0.21864216029644012 (0.23061173167468474) + | > loss_disc_real_3: 0.19566404819488525 (0.2354322779336962) + | > loss_disc_real_4: 0.24133750796318054 (0.23538319797624446) + | > loss_disc_real_5: 0.23750708997249603 (0.23393638825054638) + | > loss_0: 2.583225965499878 (2.6443063095353354) + | > grad_norm_0: tensor(10.2486, device='cuda:0') (tensor(11.2097, device='cuda:0')) + | > loss_gen: 2.0614352226257324 (2.00558285627238) + | > loss_kl: 2.408198595046997 (2.3531717589050816) + | > loss_feat: 2.943577766418457 (2.929830457499176) + | > loss_mel: 28.86360740661621 (29.003538135333127) + | > loss_duration: 1.597344994544983 (1.6137976103534735) + | > amp_scaler: 512.0 (601.3814041745735) + | > loss_1: 37.87416458129883 (37.90592083578092) + | > grad_norm_1: tensor(202.0283, device='cuda:0') (tensor(130.2824, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 1.1215 (0.9124889206388643) + | > loader_time: 0.0203 (0.0148758933485346) + + + --> TIME: 2024-06-25 03:27:24 -- STEP: 552/591 -- GLOBAL_STEP: 2325 + | > loss_disc: 2.615830183029175 (2.645294130280398) + | > loss_disc_real_0: 0.23482608795166016 (0.2190149656321476) + | > loss_disc_real_1: 0.21029111742973328 (0.22232814187156985) + | > loss_disc_real_2: 0.2637072205543518 (0.23091817752498642) + | > loss_disc_real_3: 0.20569553971290588 (0.23547485689430134) + | > loss_disc_real_4: 0.27674156427383423 (0.23534509509909843) + | > loss_disc_real_5: 0.2967712879180908 (0.23404620625618575) + | > loss_0: 2.615830183029175 (2.645294130280398) + | > grad_norm_0: tensor(7.2856, device='cuda:0') (tensor(11.3469, device='cuda:0')) + | > loss_gen: 1.7947425842285156 (2.0043018624402436) + | > loss_kl: 2.4099791049957275 (2.3505165175251337) + | > loss_feat: 2.8251051902770996 (2.9238564449807867) + | > loss_mel: 28.479982376098633 (28.935413367506392) + | > loss_duration: 1.6800003051757812 (1.6147459883620774) + | > amp_scaler: 512.0 (597.3333333333337) + | > loss_1: 37.1898078918457 (37.82883419506791) + | > grad_norm_1: tensor(216.6510, device='cuda:0') (tensor(130.9983, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 1.1025 (0.9222553182339323) + | > loader_time: 0.0261 (0.015234801216401916) + + + --> TIME: 2024-06-25 03:27:54 -- STEP: 577/591 -- GLOBAL_STEP: 2350 + | > loss_disc: 2.8712024688720703 (2.6485411054753354) + | > loss_disc_real_0: 0.1528109312057495 (0.2186658790325375) + | > loss_disc_real_1: 0.194128155708313 (0.22267613057660482) + | > loss_disc_real_2: 0.21646103262901306 (0.23128502406631843) + | > loss_disc_real_3: 0.1845792680978775 (0.23576130634060766) + | > loss_disc_real_4: 0.22364062070846558 (0.23551984012540125) + | > loss_disc_real_5: 0.2148040235042572 (0.23424231458577993) + | > loss_0: 2.8712024688720703 (2.6485411054753354) + | > grad_norm_0: tensor(18.2472, device='cuda:0') (tensor(11.3339, device='cuda:0')) + | > loss_gen: 2.1414427757263184 (2.0027877026248806) + | > loss_kl: 2.2760508060455322 (2.346566470491618) + | > loss_feat: 2.8823909759521484 (2.912728910016434) + | > loss_mel: 28.508228302001953 (28.883766270179688) + | > loss_duration: 1.6574385166168213 (1.6155769812585579) + | > amp_scaler: 512.0 (593.6360485268634) + | > loss_1: 37.465553283691406 (37.76142634717389) + | > grad_norm_1: tensor(164.2098, device='cuda:0') (tensor(131.2368, device='cuda:0')) + | > current_lr_0: 0.00019992500937460937 + | > current_lr_1: 0.00019992500937460937 + | > step_time: 1.1738 (0.9333839486748549) + | > loader_time: 0.0214 (0.015602808064769831) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035192002641393784 (-0.00033605859634724044) + | > avg_loss_disc: 3.1108811053823917 (+0.5094057296184773) + | > avg_loss_disc_real_0: 0.16087516270419386 (-0.09181727928684111) + | > avg_loss_disc_real_1: 0.258559192431734 (+0.07215284825639523) + | > avg_loss_disc_real_2: 0.28914237497968887 (+0.10015714327071584) + | > avg_loss_disc_real_3: 0.31230582360257503 (+0.11198244148746458) + | > avg_loss_disc_real_4: 0.280934731693978 (+0.03829535144440668) + | > avg_loss_disc_real_5: 0.30951338943014756 (+0.12666747195923583) + | > avg_loss_0: 3.1108811053823917 (+0.5094057296184773) + | > avg_loss_gen: 1.777586503231779 (-0.12237798660359478) + | > avg_loss_kl: 2.2737470256521344 (-0.037500158269355044) + | > avg_loss_feat: 2.493887777024127 (-0.6946476900831189) + | > avg_loss_mel: 28.435769466643638 (-2.1018007562515564) + | > avg_loss_duration: 1.5763282395423728 (-0.05262357123354633) + | > avg_loss_1: 36.55731939762196 (-3.008949847931575) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_2364.pth + + > EPOCH: 5/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 03:28:23)  + + --> TIME: 2024-06-25 03:28:34 -- STEP: 11/591 -- GLOBAL_STEP: 2375 + | > loss_disc: 2.786975145339966 (2.974070440639149) + | > loss_disc_real_0: 0.25620701909065247 (0.24156914109533484) + | > loss_disc_real_1: 0.2641953229904175 (0.3905942304567857) + | > loss_disc_real_2: 0.23891663551330566 (0.24443841251459988) + | > loss_disc_real_3: 0.260945588350296 (0.2529480633410541) + | > loss_disc_real_4: 0.24094176292419434 (0.24387729303403335) + | > loss_disc_real_5: 0.24015606939792633 (0.2551809996366501) + | > loss_0: 2.786975145339966 (2.974070440639149) + | > grad_norm_0: tensor(7.3183, device='cuda:0') (tensor(15.1823, device='cuda:0')) + | > loss_gen: 1.833358645439148 (1.8896047310395674) + | > loss_kl: 2.504737138748169 (2.572074760090221) + | > loss_feat: 2.273210048675537 (2.510062954642556) + | > loss_mel: 28.100418090820312 (28.671347531405363) + | > loss_duration: 1.4741166830062866 (1.5044377391988581) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 36.18584060668945 (37.14752786809748) + | > grad_norm_1: tensor(63.3733, device='cuda:0') (tensor(93.6546, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.7757 (0.7729693326083097) + | > loader_time: 0.0091 (0.008612567728215998) + + + --> TIME: 2024-06-25 03:28:54 -- STEP: 36/591 -- GLOBAL_STEP: 2400 + | > loss_disc: 2.6162266731262207 (2.8034867379400463) + | > loss_disc_real_0: 0.20668545365333557 (0.2180637079808447) + | > loss_disc_real_1: 0.22603297233581543 (0.294386708488067) + | > loss_disc_real_2: 0.22597508132457733 (0.24471461814310816) + | > loss_disc_real_3: 0.19730092585086823 (0.23581264871690008) + | > loss_disc_real_4: 0.18101681768894196 (0.24829751542872858) + | > loss_disc_real_5: 0.20709998905658722 (0.23778762792547545) + | > loss_0: 2.6162266731262207 (2.8034867379400463) + | > grad_norm_0: tensor(7.3182, device='cuda:0') (tensor(10.5258, device='cuda:0')) + | > loss_gen: 1.920491337776184 (1.8845088614357843) + | > loss_kl: 2.435807704925537 (2.4933064911100598) + | > loss_feat: 2.9092860221862793 (2.546546128061083) + | > loss_mel: 27.210037231445312 (28.231407589382595) + | > loss_duration: 1.5846233367919922 (1.5213073425822787) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 36.06024169921875 (36.67707633972168) + | > grad_norm_1: tensor(178.2803, device='cuda:0') (tensor(113.5114, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.802 (0.7841410570674472) + | > loader_time: 0.0095 (0.009237832493252225) + + + --> TIME: 2024-06-25 03:29:14 -- STEP: 61/591 -- GLOBAL_STEP: 2425 + | > loss_disc: 2.7166216373443604 (2.764167269722361) + | > loss_disc_real_0: 0.2774658799171448 (0.21730359808587638) + | > loss_disc_real_1: 0.28190848231315613 (0.27365894395796975) + | > loss_disc_real_2: 0.1954321712255478 (0.239518015843923) + | > loss_disc_real_3: 0.22217130661010742 (0.23568736725166195) + | > loss_disc_real_4: 0.27145031094551086 (0.24757975657455256) + | > loss_disc_real_5: 0.2553863525390625 (0.2383608085210206) + | > loss_0: 2.7166216373443604 (2.764167269722361) + | > grad_norm_0: tensor(11.3167, device='cuda:0') (tensor(11.6075, device='cuda:0')) + | > loss_gen: 1.9220092296600342 (1.9121608167398172) + | > loss_kl: 2.3677804470062256 (2.450821286342183) + | > loss_feat: 2.5422329902648926 (2.6082110834903403) + | > loss_mel: 26.959993362426758 (28.089479446411133) + | > loss_duration: 1.5600593090057373 (1.5371820574901143) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.352073669433594 (36.59785448918578) + | > grad_norm_1: tensor(179.2292, device='cuda:0') (tensor(126.4929, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.8306 (0.7951921048711558) + | > loader_time: 0.0122 (0.009821043639886574) + + + --> TIME: 2024-06-25 03:29:35 -- STEP: 86/591 -- GLOBAL_STEP: 2450 + | > loss_disc: 2.625549554824829 (2.7343794939129857) + | > loss_disc_real_0: 0.23593732714653015 (0.2162704527984525) + | > loss_disc_real_1: 0.20645293593406677 (0.26103769883859984) + | > loss_disc_real_2: 0.23926694691181183 (0.23754878757998002) + | > loss_disc_real_3: 0.21792814135551453 (0.2344963976810145) + | > loss_disc_real_4: 0.23844297230243683 (0.24365935006807016) + | > loss_disc_real_5: 0.2633645832538605 (0.23725296175757118) + | > loss_0: 2.625549554824829 (2.7343794939129857) + | > grad_norm_0: tensor(6.5770, device='cuda:0') (tensor(11.6333, device='cuda:0')) + | > loss_gen: 1.77531898021698 (1.9200218319892883) + | > loss_kl: 2.3352644443511963 (2.429845812708833) + | > loss_feat: 2.6982219219207764 (2.670652126157006) + | > loss_mel: 27.62019157409668 (27.98705247391102) + | > loss_duration: 1.574931263923645 (1.5477626129638318) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 36.003929138183594 (36.555334933968474) + | > grad_norm_1: tensor(94.4465, device='cuda:0') (tensor(130.7338, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.8379 (0.8032762782518253) + | > loader_time: 0.0119 (0.010186658349148062) + + + --> TIME: 2024-06-25 03:29:57 -- STEP: 111/591 -- GLOBAL_STEP: 2475 + | > loss_disc: 2.659783363342285 (2.7157214740375144) + | > loss_disc_real_0: 0.20152238011360168 (0.21448707912822026) + | > loss_disc_real_1: 0.23153631389141083 (0.25427063480690787) + | > loss_disc_real_2: 0.25559595227241516 (0.23538010614412325) + | > loss_disc_real_3: 0.23290827870368958 (0.23423227949722394) + | > loss_disc_real_4: 0.17413939535617828 (0.2415746877322326) + | > loss_disc_real_5: 0.23457501828670502 (0.23628481457362305) + | > loss_0: 2.659783363342285 (2.7157214740375144) + | > grad_norm_0: tensor(4.7435, device='cuda:0') (tensor(10.8599, device='cuda:0')) + | > loss_gen: 2.1972532272338867 (1.9295344836003072) + | > loss_kl: 2.2724852561950684 (2.409827468631506) + | > loss_feat: 2.8699982166290283 (2.7067680616636527) + | > loss_mel: 28.32015037536621 (27.904303696778445) + | > loss_duration: 1.5433151721954346 (1.5546470633498184) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 37.20320510864258 (36.50508086745802) + | > grad_norm_1: tensor(78.5697, device='cuda:0') (tensor(129.5312, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.8568 (0.8131950180809777) + | > loader_time: 0.0116 (0.010536906955478428) + + + --> TIME: 2024-06-25 03:30:18 -- STEP: 136/591 -- GLOBAL_STEP: 2500 + | > loss_disc: 2.662829637527466 (2.7056378844906313) + | > loss_disc_real_0: 0.12319634109735489 (0.21514799273298943) + | > loss_disc_real_1: 0.21246039867401123 (0.24963918525506457) + | > loss_disc_real_2: 0.2511032521724701 (0.23476381380768385) + | > loss_disc_real_3: 0.31533652544021606 (0.2344307268367094) + | > loss_disc_real_4: 0.26652300357818604 (0.24097395348636544) + | > loss_disc_real_5: 0.26897889375686646 (0.23593167719595573) + | > loss_0: 2.662829637527466 (2.7056378844906313) + | > grad_norm_0: tensor(18.2773, device='cuda:0') (tensor(11.4162, device='cuda:0')) + | > loss_gen: 1.8770480155944824 (1.9395150901640164) + | > loss_kl: 2.479353904724121 (2.398445830625648) + | > loss_feat: 2.985095262527466 (2.749166970743852) + | > loss_mel: 28.306333541870117 (27.88469415552476) + | > loss_duration: 1.5336930751800537 (1.555778897860471) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 37.181522369384766 (36.52760107376995) + | > grad_norm_1: tensor(134.0587, device='cuda:0') (tensor(130.2499, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.848 (0.8188949665602515) + | > loader_time: 0.0125 (0.01080613977768842) + + + --> TIME: 2024-06-25 03:30:40 -- STEP: 161/591 -- GLOBAL_STEP: 2525 + | > loss_disc: 2.5569589138031006 (2.700454377239536) + | > loss_disc_real_0: 0.19010624289512634 (0.21622760424980467) + | > loss_disc_real_1: 0.21757495403289795 (0.2455752414205799) + | > loss_disc_real_2: 0.1990903615951538 (0.23355103066619137) + | > loss_disc_real_3: 0.23854076862335205 (0.23390269575652128) + | > loss_disc_real_4: 0.22477112710475922 (0.24033824090631853) + | > loss_disc_real_5: 0.21332712471485138 (0.23586734592544367) + | > loss_0: 2.5569589138031006 (2.700454377239536) + | > grad_norm_0: tensor(7.6328, device='cuda:0') (tensor(11.7132, device='cuda:0')) + | > loss_gen: 2.277364730834961 (1.9465148219410677) + | > loss_kl: 2.2836685180664062 (2.389978327366138) + | > loss_feat: 3.306312084197998 (2.7672152874632654) + | > loss_mel: 27.888263702392578 (27.8810336900794) + | > loss_duration: 1.560640811920166 (1.553539821079799) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 37.31624984741211 (36.53828200346192) + | > grad_norm_1: tensor(142.0452, device='cuda:0') (tensor(130.4312, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.8597 (0.8244217078878271) + | > loader_time: 0.0119 (0.010990501190564642) + + + --> TIME: 2024-06-25 03:31:02 -- STEP: 186/591 -- GLOBAL_STEP: 2550 + | > loss_disc: 2.607424736022949 (2.694330780736862) + | > loss_disc_real_0: 0.23946060240268707 (0.2166914226266966) + | > loss_disc_real_1: 0.24972392618656158 (0.24303558925467145) + | > loss_disc_real_2: 0.189510777592659 (0.23345874658515375) + | > loss_disc_real_3: 0.19368556141853333 (0.23366476891822713) + | > loss_disc_real_4: 0.22245171666145325 (0.23945597855634587) + | > loss_disc_real_5: 0.23911482095718384 (0.23589436934199384) + | > loss_0: 2.607424736022949 (2.694330780736862) + | > grad_norm_0: tensor(8.3539, device='cuda:0') (tensor(12.0914, device='cuda:0')) + | > loss_gen: 1.7862846851348877 (1.9498849953374555) + | > loss_kl: 2.3806581497192383 (2.382028962976191) + | > loss_feat: 2.929483652114868 (2.787966373146222) + | > loss_mel: 26.82196617126465 (27.85933977557767) + | > loss_duration: 1.5401897430419922 (1.5508805468518247) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.458580017089844 (36.5301006891394) + | > grad_norm_1: tensor(45.6192, device='cuda:0') (tensor(131.0214, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.8619 (0.8293166660493418) + | > loader_time: 0.0114 (0.01109847586642029) + + + --> TIME: 2024-06-25 03:31:24 -- STEP: 211/591 -- GLOBAL_STEP: 2575 + | > loss_disc: 2.621101140975952 (2.689716045325402) + | > loss_disc_real_0: 0.2306712120771408 (0.21581036355611272) + | > loss_disc_real_1: 0.18996909260749817 (0.2408293639596604) + | > loss_disc_real_2: 0.19800007343292236 (0.23328074643397217) + | > loss_disc_real_3: 0.2599160373210907 (0.2337223084475757) + | > loss_disc_real_4: 0.27038413286209106 (0.2395745198732304) + | > loss_disc_real_5: 0.22194188833236694 (0.23569217825670377) + | > loss_0: 2.621101140975952 (2.689716045325402) + | > grad_norm_0: tensor(5.2544, device='cuda:0') (tensor(12.2379, device='cuda:0')) + | > loss_gen: 2.0510671138763428 (1.9562674461382825) + | > loss_kl: 2.4644296169281006 (2.3808691490318) + | > loss_feat: 2.8789801597595215 (2.804605009431523) + | > loss_mel: 27.002464294433594 (27.814310317920846) + | > loss_duration: 1.5174436569213867 (1.5488938375671892) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.91438674926758 (36.50494581828185) + | > grad_norm_1: tensor(81.1887, device='cuda:0') (tensor(127.7795, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.8792 (0.8345682135125471) + | > loader_time: 0.0127 (0.01131502146969474) + + + --> TIME: 2024-06-25 03:31:47 -- STEP: 236/591 -- GLOBAL_STEP: 2600 + | > loss_disc: 2.6931216716766357 (2.6867670416831975) + | > loss_disc_real_0: 0.15552443265914917 (0.21541540357836728) + | > loss_disc_real_1: 0.2420354187488556 (0.23926929213990594) + | > loss_disc_real_2: 0.2619222104549408 (0.23305839274899434) + | > loss_disc_real_3: 0.1890639215707779 (0.23358026052177963) + | > loss_disc_real_4: 0.22589664161205292 (0.2392530143892361) + | > loss_disc_real_5: 0.20920076966285706 (0.23607895176036883) + | > loss_0: 2.6931216716766357 (2.6867670416831975) + | > grad_norm_0: tensor(13.4850, device='cuda:0') (tensor(12.1084, device='cuda:0')) + | > loss_gen: 1.986598014831543 (1.9565764322119243) + | > loss_kl: 2.298853635787964 (2.3753091389850054) + | > loss_feat: 2.8833892345428467 (2.808177352961848) + | > loss_mel: 28.564708709716797 (27.780568551209015) + | > loss_duration: 1.5428075790405273 (1.5478312873234183) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 37.27635955810547 (36.46846287937486) + | > grad_norm_1: tensor(67.5810, device='cuda:0') (tensor(126.2231, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.8928 (0.8391964799266749) + | > loader_time: 0.0127 (0.011589784743422165) + + + --> TIME: 2024-06-25 03:32:10 -- STEP: 261/591 -- GLOBAL_STEP: 2625 + | > loss_disc: 2.7424421310424805 (2.6870005669721704) + | > loss_disc_real_0: 0.28530067205429077 (0.21617449935654115) + | > loss_disc_real_1: 0.1973942667245865 (0.23848208037144375) + | > loss_disc_real_2: 0.22196504473686218 (0.23294114392835974) + | > loss_disc_real_3: 0.22894541919231415 (0.23396292648552935) + | > loss_disc_real_4: 0.251756876707077 (0.23886085213144165) + | > loss_disc_real_5: 0.2378634363412857 (0.23585946207073913) + | > loss_0: 2.7424421310424805 (2.6870005669721704) + | > grad_norm_0: tensor(14.6759, device='cuda:0') (tensor(12.4415, device='cuda:0')) + | > loss_gen: 1.9475353956222534 (1.9588936530767274) + | > loss_kl: 2.375288486480713 (2.3724149524937204) + | > loss_feat: 2.99947190284729 (2.810770106041569) + | > loss_mel: 28.308019638061523 (27.769727823834767) + | > loss_duration: 1.5550062656402588 (1.5466706072233647) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 37.185325622558594 (36.45847722488343) + | > grad_norm_1: tensor(80.4288, device='cuda:0') (tensor(127.2255, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.8833 (0.8442331173410816) + | > loader_time: 0.0155 (0.01189475772024571) + + + --> TIME: 2024-06-25 03:32:33 -- STEP: 286/591 -- GLOBAL_STEP: 2650 + | > loss_disc: 2.7513961791992188 (2.6852869212210595) + | > loss_disc_real_0: 0.2473791539669037 (0.21576272905967037) + | > loss_disc_real_1: 0.22227543592453003 (0.2378040339563276) + | > loss_disc_real_2: 0.27926912903785706 (0.233062974431298) + | > loss_disc_real_3: 0.21053273975849152 (0.23386674238876862) + | > loss_disc_real_4: 0.21406789124011993 (0.23895137116208776) + | > loss_disc_real_5: 0.2036384791135788 (0.23554672311235975) + | > loss_0: 2.7513961791992188 (2.6852869212210595) + | > grad_norm_0: tensor(8.9601, device='cuda:0') (tensor(12.2346, device='cuda:0')) + | > loss_gen: 1.7467262744903564 (1.9598634901580276) + | > loss_kl: 2.4376392364501953 (2.368708812273467) + | > loss_feat: 2.752061367034912 (2.815535981338341) + | > loss_mel: 26.78074836730957 (27.7276588386589) + | > loss_duration: 1.5252504348754883 (1.5461757541536454) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.24242401123047 (36.41794290075768) + | > grad_norm_1: tensor(178.6830, device='cuda:0') (tensor(127.2693, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.9205 (0.8490019361455955) + | > loader_time: 0.0139 (0.01220827169351644) + + + --> TIME: 2024-06-25 03:32:56 -- STEP: 311/591 -- GLOBAL_STEP: 2675 + | > loss_disc: 2.690613269805908 (2.683047158158476) + | > loss_disc_real_0: 0.18427477777004242 (0.21540652413124822) + | > loss_disc_real_1: 0.2803645431995392 (0.23763473399007431) + | > loss_disc_real_2: 0.22902867197990417 (0.23260037821397136) + | > loss_disc_real_3: 0.27297765016555786 (0.23374957248712275) + | > loss_disc_real_4: 0.2333817034959793 (0.2383868520378683) + | > loss_disc_real_5: 0.21328100562095642 (0.23530686677843812) + | > loss_0: 2.690613269805908 (2.683047158158476) + | > grad_norm_0: tensor(9.3793, device='cuda:0') (tensor(12.2735, device='cuda:0')) + | > loss_gen: 2.0476303100585938 (1.962498646073786) + | > loss_kl: 2.2620232105255127 (2.365088686682404) + | > loss_feat: 2.878636598587036 (2.819473196839213) + | > loss_mel: 28.10422134399414 (27.696102872155485) + | > loss_duration: 1.5055735111236572 (1.545643472594847) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 36.7980842590332 (36.38880694717454) + | > grad_norm_1: tensor(250.0000, device='cuda:0') (tensor(127.7399, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.9132 (0.8542925805524227) + | > loader_time: 0.0148 (0.012506806198807004) + + + --> TIME: 2024-06-25 03:33:20 -- STEP: 336/591 -- GLOBAL_STEP: 2700 + | > loss_disc: 2.647338390350342 (2.6828809997865126) + | > loss_disc_real_0: 0.17348769307136536 (0.21579645170519748) + | > loss_disc_real_1: 0.21691973507404327 (0.2364546939553249) + | > loss_disc_real_2: 0.18165813386440277 (0.23246344821971088) + | > loss_disc_real_3: 0.22721324861049652 (0.23399557559085743) + | > loss_disc_real_4: 0.21096621453762054 (0.2379938635442938) + | > loss_disc_real_5: 0.21365362405776978 (0.23527475113847426) + | > loss_0: 2.647338390350342 (2.6828809997865126) + | > grad_norm_0: tensor(7.9104, device='cuda:0') (tensor(12.4504, device='cuda:0')) + | > loss_gen: 2.1136598587036133 (1.963673359226613) + | > loss_kl: 2.30942964553833 (2.3617478445881885) + | > loss_feat: 2.772089958190918 (2.823172450775192) + | > loss_mel: 27.80658721923828 (27.679734116508847) + | > loss_duration: 1.5313632488250732 (1.545448657657419) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 36.53313064575195 (36.37377653803142) + | > grad_norm_1: tensor(156.2727, device='cuda:0') (tensor(128.2080, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.936 (0.8597054701475867) + | > loader_time: 0.0181 (0.012764435438882735) + + + --> TIME: 2024-06-25 03:33:44 -- STEP: 361/591 -- GLOBAL_STEP: 2725 + | > loss_disc: 2.5666394233703613 (2.678070866830462) + | > loss_disc_real_0: 0.17452213168144226 (0.21513855292196088) + | > loss_disc_real_1: 0.2125980257987976 (0.2356598594347195) + | > loss_disc_real_2: 0.19995152950286865 (0.232290855033576) + | > loss_disc_real_3: 0.24668990075588226 (0.23408294925233994) + | > loss_disc_real_4: 0.2310628890991211 (0.23756208764053777) + | > loss_disc_real_5: 0.20568588376045227 (0.23514580528491752) + | > loss_0: 2.5666394233703613 (2.678070866830462) + | > grad_norm_0: tensor(9.7760, device='cuda:0') (tensor(12.0905, device='cuda:0')) + | > loss_gen: 2.1940250396728516 (1.9655806308307806) + | > loss_kl: 2.250492572784424 (2.3569123844030497) + | > loss_feat: 3.202592611312866 (2.8309964360953024) + | > loss_mel: 27.580726623535156 (27.634598634249617) + | > loss_duration: 1.613012433052063 (1.545812248192996) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 36.840850830078125 (36.33390039882498) + | > grad_norm_1: tensor(76.7119, device='cuda:0') (tensor(128.4592, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.9471 (0.8650305231522322) + | > loader_time: 0.0172 (0.013025493833166738) + + + --> TIME: 2024-06-25 03:34:08 -- STEP: 386/591 -- GLOBAL_STEP: 2750 + | > loss_disc: 2.6343836784362793 (2.6792736757604567) + | > loss_disc_real_0: 0.28775036334991455 (0.21604159837250883) + | > loss_disc_real_1: 0.1956336945295334 (0.23474908346030374) + | > loss_disc_real_2: 0.2218102514743805 (0.2321117764309898) + | > loss_disc_real_3: 0.22271664440631866 (0.23413106370620776) + | > loss_disc_real_4: 0.19179227948188782 (0.23740259168061567) + | > loss_disc_real_5: 0.22860866785049438 (0.23551063621754473) + | > loss_0: 2.6343836784362793 (2.6792736757604567) + | > grad_norm_0: tensor(10.8804, device='cuda:0') (tensor(12.4564, device='cuda:0')) + | > loss_gen: 2.0340523719787598 (1.9654993362377344) + | > loss_kl: 2.2470414638519287 (2.35671190832563) + | > loss_feat: 2.6681058406829834 (2.8303451050130803) + | > loss_mel: 26.59919548034668 (27.597368576366048) + | > loss_duration: 1.5432543754577637 (1.5462630943930828) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.091651916503906 (36.29618805801312) + | > grad_norm_1: tensor(152.4476, device='cuda:0') (tensor(129.5728, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.9678 (0.870894773636457) + | > loader_time: 0.0171 (0.013262531300282846) + + + --> TIME: 2024-06-25 03:34:33 -- STEP: 411/591 -- GLOBAL_STEP: 2775 + | > loss_disc: 2.7702724933624268 (2.677883665637088) + | > loss_disc_real_0: 0.1998564749956131 (0.21558049437193394) + | > loss_disc_real_1: 0.25382912158966064 (0.23417991087964557) + | > loss_disc_real_2: 0.27469509840011597 (0.23207402044404163) + | > loss_disc_real_3: 0.23293323814868927 (0.234174630430203) + | > loss_disc_real_4: 0.25970348715782166 (0.2374802016791346) + | > loss_disc_real_5: 0.20500977337360382 (0.2354923101672291) + | > loss_0: 2.7702724933624268 (2.677883665637088) + | > grad_norm_0: tensor(7.6345, device='cuda:0') (tensor(12.1891, device='cuda:0')) + | > loss_gen: 1.9064700603485107 (1.964429680158332) + | > loss_kl: 2.2563889026641846 (2.3530576397322682) + | > loss_feat: 2.7800750732421875 (2.827479614828624) + | > loss_mel: 27.882463455200195 (27.543307529458744) + | > loss_duration: 1.547783374786377 (1.5459315420067223) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 36.3731803894043 (36.23420604882155) + | > grad_norm_1: tensor(192.5141, device='cuda:0') (tensor(130.7704, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 1.1062 (0.8775152139130007) + | > loader_time: 0.0165 (0.013525080506819005) + + + --> TIME: 2024-06-25 03:34:59 -- STEP: 436/591 -- GLOBAL_STEP: 2800 + | > loss_disc: 2.5545454025268555 (2.6779755211751395) + | > loss_disc_real_0: 0.23945018649101257 (0.21548160107447467) + | > loss_disc_real_1: 0.20511212944984436 (0.23368853737318185) + | > loss_disc_real_2: 0.26142361760139465 (0.23196988757870612) + | > loss_disc_real_3: 0.25427374243736267 (0.2345772952251478) + | > loss_disc_real_4: 0.20317375659942627 (0.23743228262307448) + | > loss_disc_real_5: 0.2152421921491623 (0.23561148430912868) + | > loss_0: 2.5545454025268555 (2.6779755211751395) + | > grad_norm_0: tensor(9.1937, device='cuda:0') (tensor(12.2363, device='cuda:0')) + | > loss_gen: 1.9193118810653687 (1.9647805715372804) + | > loss_kl: 2.154409170150757 (2.350767176632488) + | > loss_feat: 3.010741949081421 (2.826750096924807) + | > loss_mel: 26.685487747192383 (27.507540265354542) + | > loss_duration: 1.5772149562835693 (1.5458178457317002) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.347164154052734 (36.19565599773996) + | > grad_norm_1: tensor(93.7870, device='cuda:0') (tensor(130.5690, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 0.9978 (0.8843127428938486) + | > loader_time: 0.0162 (0.01383129227052041) + + + --> TIME: 2024-06-25 03:35:24 -- STEP: 461/591 -- GLOBAL_STEP: 2825 + | > loss_disc: 2.6015827655792236 (2.6774830068269675) + | > loss_disc_real_0: 0.29906439781188965 (0.21528041472662554) + | > loss_disc_real_1: 0.21511220932006836 (0.23328883771394698) + | > loss_disc_real_2: 0.21646568179130554 (0.23175709893413843) + | > loss_disc_real_3: 0.23737511038780212 (0.23455541207578331) + | > loss_disc_real_4: 0.17968176305294037 (0.2374082898757463) + | > loss_disc_real_5: 0.20146594941616058 (0.23557582710932237) + | > loss_0: 2.6015827655792236 (2.6774830068269675) + | > grad_norm_0: tensor(16.6341, device='cuda:0') (tensor(12.1214, device='cuda:0')) + | > loss_gen: 1.9144455194473267 (1.9640934689701768) + | > loss_kl: 2.3324694633483887 (2.34752982971208) + | > loss_feat: 2.825752019882202 (2.8236219515769436) + | > loss_mel: 26.638887405395508 (27.46459812818018) + | > loss_duration: 1.5524117946624756 (1.5458730006683417) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.26396179199219 (36.1457164458235) + | > grad_norm_1: tensor(124.2497, device='cuda:0') (tensor(130.6474, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 1.0076 (0.8912307300691747) + | > loader_time: 0.0195 (0.014093238722993597) + + + --> TIME: 2024-06-25 03:35:51 -- STEP: 486/591 -- GLOBAL_STEP: 2850 + | > loss_disc: 2.6445937156677246 (2.677047591641116) + | > loss_disc_real_0: 0.2544475197792053 (0.21546889093432406) + | > loss_disc_real_1: 0.22284628450870514 (0.23283671437467565) + | > loss_disc_real_2: 0.23125870525836945 (0.23157255336213015) + | > loss_disc_real_3: 0.23018145561218262 (0.2345336544722196) + | > loss_disc_real_4: 0.2350347638130188 (0.2373838260033984) + | > loss_disc_real_5: 0.26830700039863586 (0.23553322491699777) + | > loss_0: 2.6445937156677246 (2.677047591641116) + | > grad_norm_0: tensor(7.3444, device='cuda:0') (tensor(12.1245, device='cuda:0')) + | > loss_gen: 1.9407752752304077 (1.963941423981278) + | > loss_kl: 2.2355430126190186 (2.3461009450410133) + | > loss_feat: 2.79467511177063 (2.8230287753996035) + | > loss_mel: 26.016746520996094 (27.430794382291566) + | > loss_duration: 1.5277671813964844 (1.546388896649757) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.515506744384766 (36.110254483948964) + | > grad_norm_1: tensor(86.2331, device='cuda:0') (tensor(130.8594, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 1.0618 (0.898221750318268) + | > loader_time: 0.0208 (0.01441906263798843) + + + --> TIME: 2024-06-25 03:36:18 -- STEP: 511/591 -- GLOBAL_STEP: 2875 + | > loss_disc: 2.7174763679504395 (2.677422313540882) + | > loss_disc_real_0: 0.23147496581077576 (0.21526524178144282) + | > loss_disc_real_1: 0.2167815864086151 (0.23270342068896133) + | > loss_disc_real_2: 0.23586489260196686 (0.2316454367861589) + | > loss_disc_real_3: 0.21272434294223785 (0.23445162410372158) + | > loss_disc_real_4: 0.19785551726818085 (0.23723782047833483) + | > loss_disc_real_5: 0.24428537487983704 (0.23556566524179015) + | > loss_0: 2.7174763679504395 (2.677422313540882) + | > grad_norm_0: tensor(6.3229, device='cuda:0') (tensor(12.2540, device='cuda:0')) + | > loss_gen: 2.528026819229126 (1.9658059674933) + | > loss_kl: 2.2757647037506104 (2.343607197535715) + | > loss_feat: 2.896852970123291 (2.8250537380546965) + | > loss_mel: 26.661151885986328 (27.39184810690684) + | > loss_duration: 1.5779473781585693 (1.546961107599292) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.93974304199219 (36.073276176377945) + | > grad_norm_1: tensor(160.0809, device='cuda:0') (tensor(129.9588, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 1.065 (0.9059380691811995) + | > loader_time: 0.0214 (0.014724949334698879) + + + --> TIME: 2024-06-25 03:36:46 -- STEP: 536/591 -- GLOBAL_STEP: 2900 + | > loss_disc: 2.7501564025878906 (2.6803508294162466) + | > loss_disc_real_0: 0.21519048511981964 (0.2157710590608307) + | > loss_disc_real_1: 0.17116288840770721 (0.23320812376132646) + | > loss_disc_real_2: 0.27422896027565 (0.23191997289323985) + | > loss_disc_real_3: 0.23464618623256683 (0.2346784384059372) + | > loss_disc_real_4: 0.2481711357831955 (0.23755239395063316) + | > loss_disc_real_5: 0.27281513810157776 (0.23580083606848076) + | > loss_0: 2.7501564025878906 (2.6803508294162466) + | > grad_norm_0: tensor(7.4370, device='cuda:0') (tensor(12.3923, device='cuda:0')) + | > loss_gen: 2.4238662719726562 (1.9685968658817348) + | > loss_kl: 2.238940954208374 (2.3397119352177014) + | > loss_feat: 3.5797371864318848 (2.8278494681884982) + | > loss_mel: 27.101688385009766 (27.353752360415104) + | > loss_duration: 1.5769290924072266 (1.5476101621318217) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 36.92115783691406 (36.037520871233546) + | > grad_norm_1: tensor(163.2258, device='cuda:0') (tensor(131.3828, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 1.111 (0.9159311999135941) + | > loader_time: 0.0242 (0.015050512641223509) + + + --> TIME: 2024-06-25 03:37:16 -- STEP: 561/591 -- GLOBAL_STEP: 2925 + | > loss_disc: 2.7027525901794434 (2.687996068740274) + | > loss_disc_real_0: 0.16984376311302185 (0.21590581942688336) + | > loss_disc_real_1: 0.24397286772727966 (0.2365996488594116) + | > loss_disc_real_2: 0.24149012565612793 (0.2329412608170042) + | > loss_disc_real_3: 0.21820539236068726 (0.23494085927166827) + | > loss_disc_real_4: 0.2512836456298828 (0.2376486880044036) + | > loss_disc_real_5: 0.24059270322322845 (0.23620476362539486) + | > loss_0: 2.7027525901794434 (2.687996068740274) + | > grad_norm_0: tensor(8.9124, device='cuda:0') (tensor(12.5656, device='cuda:0')) + | > loss_gen: 1.943268060684204 (1.969954641532558) + | > loss_kl: 2.157221555709839 (2.3363195574857336) + | > loss_feat: 2.524733543395996 (2.817608296552443) + | > loss_mel: 26.60041046142578 (27.332676610419668) + | > loss_duration: 1.5050396919250488 (1.548413394819181) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.730674743652344 (36.00497260408177) + | > grad_norm_1: tensor(93.0679, device='cuda:0') (tensor(131.3161, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 1.2062 (0.9268820124301467) + | > loader_time: 0.0254 (0.015410368030296502) + + + --> TIME: 2024-06-25 03:37:47 -- STEP: 586/591 -- GLOBAL_STEP: 2950 + | > loss_disc: 2.671029806137085 (2.688700912755504) + | > loss_disc_real_0: 0.1712401658296585 (0.21561335426446923) + | > loss_disc_real_1: 0.23553287982940674 (0.2364874868669607) + | > loss_disc_real_2: 0.24337522685527802 (0.23292495648506964) + | > loss_disc_real_3: 0.236461341381073 (0.235042409532713) + | > loss_disc_real_4: 0.28743085265159607 (0.23775937731152103) + | > loss_disc_real_5: 0.28741562366485596 (0.23614424484886815) + | > loss_0: 2.671029806137085 (2.688700912755504) + | > grad_norm_0: tensor(8.8174, device='cuda:0') (tensor(12.4324, device='cuda:0')) + | > loss_gen: 2.015585422515869 (1.9657612447038852) + | > loss_kl: 2.2732863426208496 (2.3310772648443545) + | > loss_feat: 2.617880344390869 (2.8018969447946374) + | > loss_mel: 25.913070678710938 (27.283916688209512) + | > loss_duration: 1.5620235204696655 (1.5492176157215756) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.3818473815918 (35.93186987789006) + | > grad_norm_1: tensor(82.8760, device='cuda:0') (tensor(130.6214, device='cuda:0')) + | > current_lr_0: 0.00019990001874843754 + | > current_lr_1: 0.00019990001874843754 + | > step_time: 1.2693 (0.9388946376156073) + | > loader_time: 0.0224 (0.01580326955879915) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038104209494083485 (+0.00029122068526897006) + | > avg_loss_disc: 2.7758863733169887 (-0.334994732065403) + | > avg_loss_disc_real_0: 0.13999431960760275 (-0.020880843096591106) + | > avg_loss_disc_real_1: 0.20589031152268675 (-0.05266888090904723) + | > avg_loss_disc_real_2: 0.19473231567981394 (-0.09441005929987492) + | > avg_loss_disc_real_3: 0.18353263146065651 (-0.1287731921419185) + | > avg_loss_disc_real_4: 0.1879922715907401 (-0.09294246010323787) + | > avg_loss_disc_real_5: 0.20763408820679846 (-0.1018793012233491) + | > avg_loss_0: 2.7758863733169887 (-0.334994732065403) + | > avg_loss_gen: 1.5262047762566424 (-0.2513817269751366) + | > avg_loss_kl: 2.2625763746018106 (-0.011170651050323777) + | > avg_loss_feat: 2.783138947283968 (+0.28925117025984104) + | > avg_loss_mel: 26.550656460701152 (-1.885113005942486) + | > avg_loss_duration: 1.5008769288976143 (-0.07545131064475852) + | > avg_loss_1: 34.62345318084068 (-1.9338662167812828) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_2955.pth + + > EPOCH: 6/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 03:38:06)  + + --> TIME: 2024-06-25 03:38:24 -- STEP: 20/591 -- GLOBAL_STEP: 2975 + | > loss_disc: 2.7133800983428955 (2.699190282821655) + | > loss_disc_real_0: 0.12911775708198547 (0.21133942231535913) + | > loss_disc_real_1: 0.2651250660419464 (0.23347481787204744) + | > loss_disc_real_2: 0.2719910144805908 (0.23211927935481072) + | > loss_disc_real_3: 0.26871779561042786 (0.24162609726190568) + | > loss_disc_real_4: 0.2668045461177826 (0.24115685671567916) + | > loss_disc_real_5: 0.26731574535369873 (0.2437603861093521) + | > loss_0: 2.7133800983428955 (2.699190282821655) + | > grad_norm_0: tensor(20.8575, device='cuda:0') (tensor(13.7254, device='cuda:0')) + | > loss_gen: 2.0712039470672607 (1.9360431134700775) + | > loss_kl: 2.4061195850372314 (2.504829800128937) + | > loss_feat: 2.8578195571899414 (2.788659656047821) + | > loss_mel: 26.57361602783203 (26.68327913284302) + | > loss_duration: 1.4459182024002075 (1.4550993621349335) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.354679107666016 (35.36791172027588) + | > grad_norm_1: tensor(203.3869, device='cuda:0') (tensor(135.9607, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.7788 (0.7756701946258545) + | > loader_time: 0.009 (0.008864474296569825) + + + --> TIME: 2024-06-25 03:38:44 -- STEP: 45/591 -- GLOBAL_STEP: 3000 + | > loss_disc: 2.6576950550079346 (2.6755317264133036) + | > loss_disc_real_0: 0.2505190372467041 (0.21138864623175727) + | > loss_disc_real_1: 0.2170199304819107 (0.22709722618261974) + | > loss_disc_real_2: 0.23464582860469818 (0.22797296875052983) + | > loss_disc_real_3: 0.22354544699192047 (0.2385619448290931) + | > loss_disc_real_4: 0.27079713344573975 (0.23790345589319864) + | > loss_disc_real_5: 0.25561466813087463 (0.23879136112001206) + | > loss_0: 2.6576950550079346 (2.6755317264133036) + | > grad_norm_0: tensor(9.2984, device='cuda:0') (tensor(13.5852, device='cuda:0')) + | > loss_gen: 1.8403809070587158 (1.9415254010094536) + | > loss_kl: 2.361938953399658 (2.4639507558610707) + | > loss_feat: 2.8813328742980957 (2.836306285858154) + | > loss_mel: 26.9417724609375 (26.675452846950954) + | > loss_duration: 1.5063765048980713 (1.4737159623040093) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.531803131103516 (35.39095170762804) + | > grad_norm_1: tensor(90.8007, device='cuda:0') (tensor(130.6996, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8116 (0.7884252389272054) + | > loader_time: 0.0099 (0.009389352798461918) + + + --> TIME: 2024-06-25 03:39:05 -- STEP: 70/591 -- GLOBAL_STEP: 3025 + | > loss_disc: 2.586480140686035 (2.6635207414627073) + | > loss_disc_real_0: 0.1764674186706543 (0.21023658396942277) + | > loss_disc_real_1: 0.20882883667945862 (0.22533333897590635) + | > loss_disc_real_2: 0.16601327061653137 (0.2269941134112222) + | > loss_disc_real_3: 0.22339627146720886 (0.23610358706542423) + | > loss_disc_real_4: 0.25084900856018066 (0.23645515676055637) + | > loss_disc_real_5: 0.1963818371295929 (0.23748764459575925) + | > loss_0: 2.586480140686035 (2.6635207414627073) + | > grad_norm_0: tensor(8.2510, device='cuda:0') (tensor(13.2572, device='cuda:0')) + | > loss_gen: 2.2311549186706543 (1.9543680446488516) + | > loss_kl: 2.420055627822876 (2.4399325915745327) + | > loss_feat: 3.105419874191284 (2.876427377973284) + | > loss_mel: 26.663589477539062 (26.682185581752233) + | > loss_duration: 1.5067864656448364 (1.4824772051402502) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.927005767822266 (35.43539112636021) + | > grad_norm_1: tensor(137.4720, device='cuda:0') (tensor(132.4520, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8147 (0.7968691723687308) + | > loader_time: 0.0108 (0.009795887129647395) + + + --> TIME: 2024-06-25 03:39:26 -- STEP: 95/591 -- GLOBAL_STEP: 3050 + | > loss_disc: 2.6758408546447754 (2.6619552712691448) + | > loss_disc_real_0: 0.17702077329158783 (0.21119810295732397) + | > loss_disc_real_1: 0.2405824214220047 (0.226336197476638) + | > loss_disc_real_2: 0.2662711441516876 (0.22740989437228756) + | > loss_disc_real_3: 0.2168816179037094 (0.23569355261953254) + | > loss_disc_real_4: 0.20565839111804962 (0.2366261490081486) + | > loss_disc_real_5: 0.27000975608825684 (0.23846435233166344) + | > loss_0: 2.6758408546447754 (2.6619552712691448) + | > grad_norm_0: tensor(10.6287, device='cuda:0') (tensor(14.1490, device='cuda:0')) + | > loss_gen: 2.093411922454834 (1.9715325782173558) + | > loss_kl: 2.354121446609497 (2.4234358812633303) + | > loss_feat: 3.027796506881714 (2.924645265779997) + | > loss_mel: 26.57614517211914 (26.702851144891035) + | > loss_duration: 1.5211600065231323 (1.492662482512625) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.572635650634766 (35.515127483167134) + | > grad_norm_1: tensor(164.1228, device='cuda:0') (tensor(133.9208, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8266 (0.8039717849932219) + | > loader_time: 0.011 (0.010065507888793944) + + + --> TIME: 2024-06-25 03:39:47 -- STEP: 120/591 -- GLOBAL_STEP: 3075 + | > loss_disc: 2.6601929664611816 (2.653541229168573) + | > loss_disc_real_0: 0.1711883246898651 (0.2102069561680158) + | > loss_disc_real_1: 0.27201369404792786 (0.2269704551746448) + | > loss_disc_real_2: 0.2602827548980713 (0.2273304151992003) + | > loss_disc_real_3: 0.24535934627056122 (0.23541538106898466) + | > loss_disc_real_4: 0.26769644021987915 (0.23584452544649442) + | > loss_disc_real_5: 0.2530694603919983 (0.23753336444497108) + | > loss_0: 2.6601929664611816 (2.653541229168573) + | > grad_norm_0: tensor(9.2682, device='cuda:0') (tensor(13.3344, device='cuda:0')) + | > loss_gen: 1.9624106884002686 (1.9777121106783548) + | > loss_kl: 2.371727228164673 (2.4092746615409832) + | > loss_feat: 2.967442274093628 (2.952580295006434) + | > loss_mel: 26.660842895507812 (26.70223749478658) + | > loss_duration: 1.5319159030914307 (1.4958550065755847) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.49433898925781 (35.5376596768697) + | > grad_norm_1: tensor(205.5799, device='cuda:0') (tensor(131.7381, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8621 (0.8117692788441976) + | > loader_time: 0.011 (0.0104141632715861) + + + --> TIME: 2024-06-25 03:40:09 -- STEP: 145/591 -- GLOBAL_STEP: 3100 + | > loss_disc: 2.641019821166992 (2.647679560759971) + | > loss_disc_real_0: 0.3706490993499756 (0.21123505152505023) + | > loss_disc_real_1: 0.15695282816886902 (0.22529631945593603) + | > loss_disc_real_2: 0.14691473543643951 (0.226211728001463) + | > loss_disc_real_3: 0.12753218412399292 (0.2348691118174586) + | > loss_disc_real_4: 0.20008979737758636 (0.23548614670490398) + | > loss_disc_real_5: 0.15665028989315033 (0.23685800710628774) + | > loss_0: 2.641019821166992 (2.647679560759971) + | > grad_norm_0: tensor(30.4471, device='cuda:0') (tensor(13.5130, device='cuda:0')) + | > loss_gen: 1.9796124696731567 (1.982563226798485) + | > loss_kl: 2.412010669708252 (2.40175403233232) + | > loss_feat: 3.288048505783081 (2.974560344630275) + | > loss_mel: 26.434690475463867 (26.669279690446526) + | > loss_duration: 1.5191256999969482 (1.495970104480612) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.63349151611328 (35.52412761819773) + | > grad_norm_1: tensor(182.2147, device='cuda:0') (tensor(130.1588, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8839 (0.8181722197039374) + | > loader_time: 0.0112 (0.010666141838862976) + + + --> TIME: 2024-06-25 03:40:31 -- STEP: 170/591 -- GLOBAL_STEP: 3125 + | > loss_disc: 2.5669496059417725 (2.644602465629577) + | > loss_disc_real_0: 0.22646290063858032 (0.2116169463185703) + | > loss_disc_real_1: 0.24814090132713318 (0.22509081389974145) + | > loss_disc_real_2: 0.2955792248249054 (0.22599835685070824) + | > loss_disc_real_3: 0.27637988328933716 (0.235305234263925) + | > loss_disc_real_4: 0.26552966237068176 (0.23535364524406546) + | > loss_disc_real_5: 0.22605715692043304 (0.23655359981691135) + | > loss_0: 2.5669496059417725 (2.644602465629577) + | > grad_norm_0: tensor(8.2200, device='cuda:0') (tensor(13.5636, device='cuda:0')) + | > loss_gen: 1.6950409412384033 (1.9844045351533328) + | > loss_kl: 2.3472273349761963 (2.392707699887891) + | > loss_feat: 3.13752818107605 (2.9847274808322686) + | > loss_mel: 26.21811294555664 (26.627882643306958) + | > loss_duration: 1.4777085781097412 (1.493788782989277) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.87561798095703 (35.48351139741786) + | > grad_norm_1: tensor(52.5800, device='cuda:0') (tensor(132.2116, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8821 (0.8248171666089228) + | > loader_time: 0.0125 (0.010905006352592917) + + + --> TIME: 2024-06-25 03:40:53 -- STEP: 195/591 -- GLOBAL_STEP: 3150 + | > loss_disc: 2.6272528171539307 (2.6458947878617494) + | > loss_disc_real_0: 0.23598089814186096 (0.21229120710721383) + | > loss_disc_real_1: 0.17505775392055511 (0.2250617685990456) + | > loss_disc_real_2: 0.17972247302532196 (0.22576429713995028) + | > loss_disc_real_3: 0.17873787879943848 (0.23464657564957936) + | > loss_disc_real_4: 0.2076539844274521 (0.23529837299615908) + | > loss_disc_real_5: 0.20962224900722504 (0.23676669330168992) + | > loss_0: 2.6272528171539307 (2.6458947878617494) + | > grad_norm_0: tensor(8.6358, device='cuda:0') (tensor(13.4064, device='cuda:0')) + | > loss_gen: 2.0962748527526855 (1.987324751340426) + | > loss_kl: 2.353177785873413 (2.387414127741101) + | > loss_feat: 3.061175584793091 (2.9891531222905874) + | > loss_mel: 26.605167388916016 (26.608100196642752) + | > loss_duration: 1.4849668741226196 (1.492654761901268) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.60076141357422 (35.4646471854968) + | > grad_norm_1: tensor(84.7534, device='cuda:0') (tensor(133.2889, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8567 (0.8298819737556654) + | > loader_time: 0.0129 (0.011089375080206455) + + + --> TIME: 2024-06-25 03:41:15 -- STEP: 220/591 -- GLOBAL_STEP: 3175 + | > loss_disc: 2.5993752479553223 (2.6435405406084915) + | > loss_disc_real_0: 0.1710597276687622 (0.21165098263458773) + | > loss_disc_real_1: 0.2570634186267853 (0.2251200487667864) + | > loss_disc_real_2: 0.2582888901233673 (0.22577559487386184) + | > loss_disc_real_3: 0.24822290241718292 (0.23455711488019337) + | > loss_disc_real_4: 0.24146446585655212 (0.23546330427581613) + | > loss_disc_real_5: 0.25001850724220276 (0.236452663351189) + | > loss_0: 2.5993752479553223 (2.6435405406084915) + | > grad_norm_0: tensor(8.9712, device='cuda:0') (tensor(13.0392, device='cuda:0')) + | > loss_gen: 1.9816553592681885 (1.9880387089469216) + | > loss_kl: 2.250654935836792 (2.382485360449009) + | > loss_feat: 2.9756717681884766 (2.9939524553038863) + | > loss_mel: 26.50205421447754 (26.603733374855736) + | > loss_duration: 1.4604601860046387 (1.4915836941112166) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.17049789428711 (35.459793767062116) + | > grad_norm_1: tensor(63.7620, device='cuda:0') (tensor(130.7941, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8878 (0.8348989486694337) + | > loader_time: 0.014 (0.011320323293859309) + + + --> TIME: 2024-06-25 03:41:38 -- STEP: 245/591 -- GLOBAL_STEP: 3200 + | > loss_disc: 2.5980043411254883 (2.6435807627074563) + | > loss_disc_real_0: 0.22667208313941956 (0.21189130213187665) + | > loss_disc_real_1: 0.17853859066963196 (0.22489852625496534) + | > loss_disc_real_2: 0.18803226947784424 (0.22503086699514974) + | > loss_disc_real_3: 0.21873685717582703 (0.2343682911931252) + | > loss_disc_real_4: 0.24479618668556213 (0.23564988599748027) + | > loss_disc_real_5: 0.1476464420557022 (0.23657884962704717) + | > loss_0: 2.5980043411254883 (2.6435807627074563) + | > grad_norm_0: tensor(7.4978, device='cuda:0') (tensor(13.2975, device='cuda:0')) + | > loss_gen: 2.1743786334991455 (1.9897128732836977) + | > loss_kl: 2.318824052810669 (2.380173783399619) + | > loss_feat: 3.0686659812927246 (2.998915982732968) + | > loss_mel: 26.27851676940918 (26.596655133305763) + | > loss_duration: 1.500837802886963 (1.4915078489147882) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.341224670410156 (35.45696583961955) + | > grad_norm_1: tensor(164.3240, device='cuda:0') (tensor(132.6623, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8818 (0.8410846291756144) + | > loader_time: 0.0131 (0.011614521182313257) + + + --> TIME: 2024-06-25 03:42:01 -- STEP: 270/591 -- GLOBAL_STEP: 3225 + | > loss_disc: 2.628807783126831 (2.6441503930974886) + | > loss_disc_real_0: 0.18947048485279083 (0.2117325191696485) + | > loss_disc_real_1: 0.20123465359210968 (0.22494852007539184) + | > loss_disc_real_2: 0.25164979696273804 (0.2251918409709577) + | > loss_disc_real_3: 0.21931087970733643 (0.23470250572319384) + | > loss_disc_real_4: 0.24066810309886932 (0.2357319289335498) + | > loss_disc_real_5: 0.1977241039276123 (0.23706171976195442) + | > loss_0: 2.628807783126831 (2.6441503930974886) + | > grad_norm_0: tensor(7.6752, device='cuda:0') (tensor(13.2503, device='cuda:0')) + | > loss_gen: 1.7657947540283203 (1.9890357308917574) + | > loss_kl: 2.375270366668701 (2.3748840914832203) + | > loss_feat: 2.90075421333313 (2.99940021302965) + | > loss_mel: 25.87021255493164 (26.56508656254521) + | > loss_duration: 1.5037305355072021 (1.491329782980459) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.41576385498047 (35.41973652309842) + | > grad_norm_1: tensor(200.7422, device='cuda:0') (tensor(132.2436, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.8919 (0.8467276873411956) + | > loader_time: 0.0154 (0.011911486696313927) + + + --> TIME: 2024-06-25 03:42:24 -- STEP: 295/591 -- GLOBAL_STEP: 3250 + | > loss_disc: 2.794039487838745 (2.648676782543377) + | > loss_disc_real_0: 0.39427897334098816 (0.2120089934791549) + | > loss_disc_real_1: 0.1659977287054062 (0.2264687763937449) + | > loss_disc_real_2: 0.207433819770813 (0.2257399208969989) + | > loss_disc_real_3: 0.20028968155384064 (0.2349459829977003) + | > loss_disc_real_4: 0.21946066617965698 (0.23569009127253193) + | > loss_disc_real_5: 0.19766420125961304 (0.2368691357515626) + | > loss_0: 2.794039487838745 (2.648676782543377) + | > grad_norm_0: tensor(34.8064, device='cuda:0') (tensor(13.5005, device='cuda:0')) + | > loss_gen: 2.025113582611084 (1.9966944540961313) + | > loss_kl: 2.2998862266540527 (2.3698874578637574) + | > loss_feat: 2.890742301940918 (3.0052585375511045) + | > loss_mel: 26.2073917388916 (26.56817526736502) + | > loss_duration: 1.4810683727264404 (1.4914972996307627) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.90420150756836 (35.43151317531779) + | > grad_norm_1: tensor(159.8903, device='cuda:0') (tensor(135.7771, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.9233 (0.8522609096462443) + | > loader_time: 0.0158 (0.012181551981780486) + + + --> TIME: 2024-06-25 03:42:48 -- STEP: 320/591 -- GLOBAL_STEP: 3275 + | > loss_disc: 2.7263641357421875 (2.667053191363811) + | > loss_disc_real_0: 0.24310004711151123 (0.21227182548027484) + | > loss_disc_real_1: 0.282702773809433 (0.23188972915522754) + | > loss_disc_real_2: 0.2605704665184021 (0.22792559796944262) + | > loss_disc_real_3: 0.21640822291374207 (0.2354456747416407) + | > loss_disc_real_4: 0.2240961641073227 (0.23742399206385018) + | > loss_disc_real_5: 0.2346508949995041 (0.23713867543265224) + | > loss_0: 2.7263641357421875 (2.667053191363811) + | > grad_norm_0: tensor(7.3508, device='cuda:0') (tensor(13.5091, device='cuda:0')) + | > loss_gen: 1.922304391860962 (1.9946788575500254) + | > loss_kl: 2.467102527618408 (2.365374192595479) + | > loss_feat: 2.4545626640319824 (2.9791944585740566) + | > loss_mel: 26.299274444580078 (26.578210723400115) + | > loss_duration: 1.4779291152954102 (1.4918164219707246) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.62117385864258 (35.40927480459214) + | > grad_norm_1: tensor(135.2230, device='cuda:0') (tensor(135.1101, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.9056 (0.857768714427948) + | > loader_time: 0.0167 (0.012465003132820127) + + + --> TIME: 2024-06-25 03:43:12 -- STEP: 345/591 -- GLOBAL_STEP: 3300 + | > loss_disc: 2.58945894241333 (2.668295384835504) + | > loss_disc_real_0: 0.15753909945487976 (0.21242984520352404) + | > loss_disc_real_1: 0.20230436325073242 (0.2318984385849773) + | > loss_disc_real_2: 0.18436644971370697 (0.22821328799793686) + | > loss_disc_real_3: 0.21622131764888763 (0.2354027335194574) + | > loss_disc_real_4: 0.21921508014202118 (0.23742574280586795) + | > loss_disc_real_5: 0.2406882345676422 (0.2370841271203497) + | > loss_0: 2.58945894241333 (2.668295384835504) + | > grad_norm_0: tensor(10.3359, device='cuda:0') (tensor(13.4724, device='cuda:0')) + | > loss_gen: 2.087366819381714 (1.9890897215276528) + | > loss_kl: 2.387887716293335 (2.360529211293094) + | > loss_feat: 2.9895455837249756 (2.9614281923874572) + | > loss_mel: 26.05352210998535 (26.548811677573383) + | > loss_duration: 1.5456863641738892 (1.4926130792369003) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.06400680541992 (35.35247201228486) + | > grad_norm_1: tensor(135.1180, device='cuda:0') (tensor(135.2744, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.9822 (0.8632960651231849) + | > loader_time: 0.0165 (0.012726452039635697) + + + --> TIME: 2024-06-25 03:43:36 -- STEP: 370/591 -- GLOBAL_STEP: 3325 + | > loss_disc: 2.6231319904327393 (2.667538050058724) + | > loss_disc_real_0: 0.24758204817771912 (0.2127841670770903) + | > loss_disc_real_1: 0.2509905993938446 (0.231271255419061) + | > loss_disc_real_2: 0.2103486955165863 (0.22819794944009264) + | > loss_disc_real_3: 0.23946109414100647 (0.23537188651593957) + | > loss_disc_real_4: 0.23662585020065308 (0.237083654468124) + | > loss_disc_real_5: 0.23370938003063202 (0.23681723958737141) + | > loss_0: 2.6231319904327393 (2.667538050058724) + | > grad_norm_0: tensor(9.0065, device='cuda:0') (tensor(13.4328, device='cuda:0')) + | > loss_gen: 1.862595796585083 (1.9858029513745699) + | > loss_kl: 2.340353012084961 (2.3592744170008446) + | > loss_feat: 2.8262224197387695 (2.952094914462116) + | > loss_mel: 25.408510208129883 (26.50416324203079) + | > loss_duration: 1.485437273979187 (1.4939113729708895) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.92311477661133 (35.29524703154692) + | > grad_norm_1: tensor(72.1659, device='cuda:0') (tensor(135.3291, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.955 (0.868744767034376) + | > loader_time: 0.0178 (0.01299589582391687) + + + --> TIME: 2024-06-25 03:44:01 -- STEP: 395/591 -- GLOBAL_STEP: 3350 + | > loss_disc: 2.545372247695923 (2.665907282165333) + | > loss_disc_real_0: 0.17936722934246063 (0.21236380210405664) + | > loss_disc_real_1: 0.23574940860271454 (0.23071642510498627) + | > loss_disc_real_2: 0.167452871799469 (0.2280005827734742) + | > loss_disc_real_3: 0.17526118457317352 (0.23531870728806603) + | > loss_disc_real_4: 0.19780531525611877 (0.23686723445035235) + | > loss_disc_real_5: 0.17874284088611603 (0.23653529999376852) + | > loss_0: 2.545372247695923 (2.665907282165333) + | > grad_norm_0: tensor(9.0321, device='cuda:0') (tensor(13.0914, device='cuda:0')) + | > loss_gen: 2.4406485557556152 (1.9847260128093673) + | > loss_kl: 2.2700746059417725 (2.356145039087607) + | > loss_feat: 3.3194963932037354 (2.9485034876231913) + | > loss_mel: 26.510358810424805 (26.47642944915385) + | > loss_duration: 1.5109095573425293 (1.494475054740905) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 36.05148696899414 (35.26027913153925) + | > grad_norm_1: tensor(97.3488, device='cuda:0') (tensor(135.8427, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.9539 (0.8748673257948477) + | > loader_time: 0.0178 (0.013269584993772867) + + + --> TIME: 2024-06-25 03:44:26 -- STEP: 420/591 -- GLOBAL_STEP: 3375 + | > loss_disc: 2.6893978118896484 (2.666023892448061) + | > loss_disc_real_0: 0.16964207589626312 (0.21248764940315767) + | > loss_disc_real_1: 0.2219696193933487 (0.23030262557523593) + | > loss_disc_real_2: 0.27046999335289 (0.22822139515053658) + | > loss_disc_real_3: 0.1900547295808792 (0.23557793451916603) + | > loss_disc_real_4: 0.22201620042324066 (0.23688188000094323) + | > loss_disc_real_5: 0.21167945861816406 (0.23671253461922917) + | > loss_0: 2.6893978118896484 (2.666023892448061) + | > grad_norm_0: tensor(10.0744, device='cuda:0') (tensor(13.2572, device='cuda:0')) + | > loss_gen: 1.9103792905807495 (1.9826183949198044) + | > loss_kl: 2.2974562644958496 (2.355000118982223) + | > loss_feat: 2.667436122894287 (2.9436358849207567) + | > loss_mel: 25.34394645690918 (26.430700965154738) + | > loss_duration: 1.500032901763916 (1.495009436777659) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.7192497253418 (35.20696487426757) + | > grad_norm_1: tensor(164.8837, device='cuda:0') (tensor(134.6037, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.9734 (0.8807415008544922) + | > loader_time: 0.02 (0.013545549483526318) + + + --> TIME: 2024-06-25 03:44:51 -- STEP: 445/591 -- GLOBAL_STEP: 3400 + | > loss_disc: 2.6161017417907715 (2.6652744025326838) + | > loss_disc_real_0: 0.2421053797006607 (0.21285509810353934) + | > loss_disc_real_1: 0.28360918164253235 (0.22984832675939196) + | > loss_disc_real_2: 0.21525712311267853 (0.22801407615790206) + | > loss_disc_real_3: 0.22752375900745392 (0.23544256415929687) + | > loss_disc_real_4: 0.2899971008300781 (0.23684055758326242) + | > loss_disc_real_5: 0.26815587282180786 (0.23653942175795523) + | > loss_0: 2.6161017417907715 (2.6652744025326838) + | > grad_norm_0: tensor(8.6555, device='cuda:0') (tensor(13.2894, device='cuda:0')) + | > loss_gen: 2.132986307144165 (1.9821730897667704) + | > loss_kl: 2.1774933338165283 (2.3524493410346214) + | > loss_feat: 3.1121222972869873 (2.9422680394033374) + | > loss_mel: 25.984800338745117 (26.402708079305928) + | > loss_duration: 1.534329891204834 (1.4957878699463398) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.94173049926758 (35.1753864845533) + | > grad_norm_1: tensor(174.7746, device='cuda:0') (tensor(134.9368, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.9976 (0.8873610866203737) + | > loader_time: 0.0158 (0.013844636317049517) + + + --> TIME: 2024-06-25 03:45:17 -- STEP: 470/591 -- GLOBAL_STEP: 3425 + | > loss_disc: 2.8789682388305664 (2.6664510189218715) + | > loss_disc_real_0: 0.22122766077518463 (0.21330208025397138) + | > loss_disc_real_1: 0.2795754075050354 (0.2295708733353209) + | > loss_disc_real_2: 0.23052190244197845 (0.22794421115454208) + | > loss_disc_real_3: 0.26312726736068726 (0.23559594512619872) + | > loss_disc_real_4: 0.25207313895225525 (0.23678103916822596) + | > loss_disc_real_5: 0.22170767188072205 (0.23640734375791347) + | > loss_0: 2.8789682388305664 (2.6664510189218715) + | > grad_norm_0: tensor(8.8717, device='cuda:0') (tensor(13.4397, device='cuda:0')) + | > loss_gen: 2.2843708992004395 (1.982621588098242) + | > loss_kl: 2.22784161567688 (2.3501728636153203) + | > loss_feat: 2.7651312351226807 (2.9402559802887294) + | > loss_mel: 26.731201171875 (26.362485889678307) + | > loss_duration: 1.5506772994995117 (1.4966524603519027) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.55922317504883 (35.13218881322981) + | > grad_norm_1: tensor(114.1742, device='cuda:0') (tensor(135.3998, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 0.9997 (0.8944832035835754) + | > loader_time: 0.0157 (0.014088503350602818) + + + --> TIME: 2024-06-25 03:45:44 -- STEP: 495/591 -- GLOBAL_STEP: 3450 + | > loss_disc: 2.6769590377807617 (2.6698577693014425) + | > loss_disc_real_0: 0.2763850688934326 (0.21341864117468246) + | > loss_disc_real_1: 0.23569037020206451 (0.23046262450892516) + | > loss_disc_real_2: 0.2181771844625473 (0.22881502421817393) + | > loss_disc_real_3: 0.26919472217559814 (0.2357478980163131) + | > loss_disc_real_4: 0.23057053983211517 (0.23695544127864068) + | > loss_disc_real_5: 0.2886618971824646 (0.23657997271629294) + | > loss_0: 2.6769590377807617 (2.6698577693014425) + | > grad_norm_0: tensor(13.8018, device='cuda:0') (tensor(13.3692, device='cuda:0')) + | > loss_gen: 2.0531911849975586 (1.9806308568126025) + | > loss_kl: 2.2945003509521484 (2.3486675604425296) + | > loss_feat: 2.7389400005340576 (2.9306315874812583) + | > loss_mel: 26.011516571044922 (26.347292254669497) + | > loss_duration: 1.5167715549468994 (1.4977984228519472) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.61492156982422 (35.105020742705356) + | > grad_norm_1: tensor(80.6102, device='cuda:0') (tensor(134.4070, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 1.0801 (0.9022354424601854) + | > loader_time: 0.0167 (0.014309991008103495) + + + --> TIME: 2024-06-25 03:46:12 -- STEP: 520/591 -- GLOBAL_STEP: 3475 + | > loss_disc: 2.693669319152832 (2.6697409982864664) + | > loss_disc_real_0: 0.28769364953041077 (0.21343875630543782) + | > loss_disc_real_1: 0.23147748410701752 (0.23016278703625387) + | > loss_disc_real_2: 0.18922577798366547 (0.22879633932159496) + | > loss_disc_real_3: 0.29409268498420715 (0.23584843022892107) + | > loss_disc_real_4: 0.24887727200984955 (0.2368302254149547) + | > loss_disc_real_5: 0.219030499458313 (0.23647390864789486) + | > loss_0: 2.693669319152832 (2.6697409982864664) + | > grad_norm_0: tensor(15.8161, device='cuda:0') (tensor(13.3652, device='cuda:0')) + | > loss_gen: 1.9856045246124268 (1.9789041787385941) + | > loss_kl: 2.2604377269744873 (2.3468431601157556) + | > loss_feat: 2.957249402999878 (2.9251376317097595) + | > loss_mel: 26.31275177001953 (26.30793199172387) + | > loss_duration: 1.5195629596710205 (1.4991151261788143) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.035606384277344 (35.05793215678287) + | > grad_norm_1: tensor(122.7890, device='cuda:0') (tensor(133.5135, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 1.1357 (0.9120651052548335) + | > loader_time: 0.0253 (0.014616333979826706) + + + --> TIME: 2024-06-25 03:46:41 -- STEP: 545/591 -- GLOBAL_STEP: 3500 + | > loss_disc: 2.624258518218994 (2.6700019591445217) + | > loss_disc_real_0: 0.3007187247276306 (0.21375895974422818) + | > loss_disc_real_1: 0.19523003697395325 (0.2297661213426415) + | > loss_disc_real_2: 0.21083548665046692 (0.2286025411218678) + | > loss_disc_real_3: 0.22628042101860046 (0.2357239546032127) + | > loss_disc_real_4: 0.21100614964962006 (0.2365616800861621) + | > loss_disc_real_5: 0.25462400913238525 (0.23635236167032783) + | > loss_0: 2.624258518218994 (2.6700019591445217) + | > grad_norm_0: tensor(11.8810, device='cuda:0') (tensor(13.4642, device='cuda:0')) + | > loss_gen: 2.255815267562866 (1.9779369686721662) + | > loss_kl: 2.2132508754730225 (2.343603163465447) + | > loss_feat: 2.952754020690918 (2.9210243675686893) + | > loss_mel: 25.918432235717773 (26.267221996762334) + | > loss_duration: 1.4983901977539062 (1.500423269096864) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.83864212036133 (35.01020981674894) + | > grad_norm_1: tensor(190.0561, device='cuda:0') (tensor(132.8654, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 1.1739 (0.9222675192246743) + | > loader_time: 0.0193 (0.01490111875971523) + + + --> TIME: 2024-06-25 03:47:11 -- STEP: 570/591 -- GLOBAL_STEP: 3525 + | > loss_disc: 2.635483980178833 (2.6700570558246803) + | > loss_disc_real_0: 0.2397317737340927 (0.21382941618692455) + | > loss_disc_real_1: 0.26158246397972107 (0.2295279262358682) + | > loss_disc_real_2: 0.2857154905796051 (0.22862991708935348) + | > loss_disc_real_3: 0.24058322608470917 (0.23580250402814465) + | > loss_disc_real_4: 0.2571655511856079 (0.23665296558225363) + | > loss_disc_real_5: 0.23245881497859955 (0.23632565331563615) + | > loss_0: 2.635483980178833 (2.6700570558246803) + | > grad_norm_0: tensor(10.8801, device='cuda:0') (tensor(13.2517, device='cuda:0')) + | > loss_gen: 1.9490809440612793 (1.9761533774827655) + | > loss_kl: 2.24837589263916 (2.340740052022431) + | > loss_feat: 2.936983108520508 (2.915166623968826) + | > loss_mel: 25.71180534362793 (26.223734902499004) + | > loss_duration: 1.5299532413482666 (1.5017074449020513) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.376197814941406 (34.95750246549909) + | > grad_norm_1: tensor(184.2078, device='cuda:0') (tensor(132.7786, device='cuda:0')) + | > current_lr_0: 0.00019987503124609398 + | > current_lr_1: 0.00019987503124609398 + | > step_time: 1.2094 (0.9332325705310754) + | > loader_time: 0.0199 (0.0152086663664433) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036077753026434716 (-0.0002026456467648769) + | > avg_loss_disc: 2.597415827690287 (-0.17847054562670195) + | > avg_loss_disc_real_0: 0.15732832363945368 (+0.017334004031850925) + | > avg_loss_disc_real_1: 0.16983682741510112 (-0.03605348410758563) + | > avg_loss_disc_real_2: 0.19380588036902405 (-0.000926435310789897) + | > avg_loss_disc_real_3: 0.23206675781848582 (+0.048534126357829305) + | > avg_loss_disc_real_4: 0.23969111639134427 (+0.05169884480060416) + | > avg_loss_disc_real_5: 0.22188038775261412 (+0.01424629954581566) + | > avg_loss_0: 2.597415827690287 (-0.17847054562670195) + | > avg_loss_gen: 1.7779513217033225 (+0.25174654544668007) + | > avg_loss_kl: 2.2878589148217046 (+0.025282540219893956) + | > avg_loss_feat: 2.92107115907872 (+0.13793221179475212) + | > avg_loss_mel: 25.815546116930374 (-0.7351103437707778) + | > avg_loss_duration: 1.4768919488216967 (-0.02398498007591754) + | > avg_loss_1: 34.27931964143794 (-0.3441335394027405) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_3546.pth + + > EPOCH: 7/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 03:47:50)  + + --> TIME: 2024-06-25 03:47:55 -- STEP: 4/591 -- GLOBAL_STEP: 3550 + | > loss_disc: 2.6428189277648926 (2.658558428287506) + | > loss_disc_real_0: 0.27005982398986816 (0.23964154720306396) + | > loss_disc_real_1: 0.21315331757068634 (0.21772610396146774) + | > loss_disc_real_2: 0.1905030757188797 (0.21181323751807213) + | > loss_disc_real_3: 0.20950165390968323 (0.24349486455321312) + | > loss_disc_real_4: 0.23957549035549164 (0.25160831212997437) + | > loss_disc_real_5: 0.20962513983249664 (0.24131488427519798) + | > loss_0: 2.6428189277648926 (2.658558428287506) + | > grad_norm_0: tensor(14.1477, device='cuda:0') (tensor(14.6588, device='cuda:0')) + | > loss_gen: 1.9836345911026 (2.002352863550186) + | > loss_kl: 2.690793752670288 (2.587571442127228) + | > loss_feat: 3.1408438682556152 (3.2025784254074097) + | > loss_mel: 27.192720413208008 (27.068668365478516) + | > loss_duration: 1.453798770904541 (1.4417183101177216) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 36.4617919921875 (36.30288887023926) + | > grad_norm_1: tensor(86.3342, device='cuda:0') (tensor(151.8153, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.7649 (0.7703111171722412) + | > loader_time: 0.0098 (0.00852590799331665) + + + --> TIME: 2024-06-25 03:48:15 -- STEP: 29/591 -- GLOBAL_STEP: 3575 + | > loss_disc: 2.768832206726074 (2.739723665960904) + | > loss_disc_real_0: 0.20456716418266296 (0.21892283471493884) + | > loss_disc_real_1: 0.2742273807525635 (0.24692772174703664) + | > loss_disc_real_2: 0.22547882795333862 (0.2354124722809627) + | > loss_disc_real_3: 0.2126326560974121 (0.23750397049147506) + | > loss_disc_real_4: 0.24347811937332153 (0.24327450219927163) + | > loss_disc_real_5: 0.26460909843444824 (0.24295079297032848) + | > loss_0: 2.768832206726074 (2.739723665960904) + | > grad_norm_0: tensor(5.6934, device='cuda:0') (tensor(13.3925, device='cuda:0')) + | > loss_gen: 1.961564064025879 (2.0059374653059865) + | > loss_kl: 2.609186887741089 (2.5242891558285416) + | > loss_feat: 2.602123260498047 (3.014040840083155) + | > loss_mel: 25.48921775817871 (26.24051646528573) + | > loss_duration: 1.4649841785430908 (1.4345425655101907) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.1270751953125 (35.219326282369664) + | > grad_norm_1: tensor(131.2007, device='cuda:0') (tensor(130.4141, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.8405 (0.7856131833175133) + | > loader_time: 0.0108 (0.00929238878447434) + + + --> TIME: 2024-06-25 03:48:37 -- STEP: 54/591 -- GLOBAL_STEP: 3600 + | > loss_disc: 2.6782429218292236 (2.730293560911108) + | > loss_disc_real_0: 0.28669875860214233 (0.222442207099111) + | > loss_disc_real_1: 0.20525725185871124 (0.23718403886865685) + | > loss_disc_real_2: 0.1933208853006363 (0.23302332780979298) + | > loss_disc_real_3: 0.19221626222133636 (0.24049014184210035) + | > loss_disc_real_4: 0.24471279978752136 (0.24351739745449136) + | > loss_disc_real_5: 0.23691117763519287 (0.23700289400639357) + | > loss_0: 2.6782429218292236 (2.730293560911108) + | > grad_norm_0: tensor(12.2522, device='cuda:0') (tensor(15.0099, device='cuda:0')) + | > loss_gen: 1.8283137083053589 (1.9700781129024647) + | > loss_kl: 2.3867039680480957 (2.47547126699377) + | > loss_feat: 2.9957363605499268 (2.9337608593481557) + | > loss_mel: 26.01393699645996 (25.98315136520951) + | > loss_duration: 1.4247350692749023 (1.452014770772722) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.6494255065918 (34.8144763663963) + | > grad_norm_1: tensor(98.7674, device='cuda:0') (tensor(123.4620, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.8295 (0.8205275668038262) + | > loader_time: 0.0108 (0.012540322762948496) + + + --> TIME: 2024-06-25 03:48:58 -- STEP: 79/591 -- GLOBAL_STEP: 3625 + | > loss_disc: 2.682140827178955 (2.6983840888059594) + | > loss_disc_real_0: 0.33479249477386475 (0.21984167698817914) + | > loss_disc_real_1: 0.2023482471704483 (0.2322269378206398) + | > loss_disc_real_2: 0.19945912063121796 (0.22909337244456326) + | > loss_disc_real_3: 0.25483182072639465 (0.23852526896362064) + | > loss_disc_real_4: 0.19077496230602264 (0.23927112150041363) + | > loss_disc_real_5: 0.2604847252368927 (0.23661414354662352) + | > loss_0: 2.682140827178955 (2.6983840888059594) + | > grad_norm_0: tensor(21.9447, device='cuda:0') (tensor(13.9417, device='cuda:0')) + | > loss_gen: 2.0207645893096924 (1.9775652885437012) + | > loss_kl: 2.5462348461151123 (2.453121478044534) + | > loss_feat: 3.191591501235962 (2.9865411323837088) + | > loss_mel: 25.807279586791992 (25.955320140983485) + | > loss_duration: 1.4538202285766602 (1.4612672494936594) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.01968765258789 (34.83381541771224) + | > grad_norm_1: tensor(170.3700, device='cuda:0') (tensor(123.7044, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.8392 (0.8230408203752734) + | > loader_time: 0.0116 (0.012101408801501309) + + + --> TIME: 2024-06-25 03:49:20 -- STEP: 104/591 -- GLOBAL_STEP: 3650 + | > loss_disc: 2.6284778118133545 (2.682815535710408) + | > loss_disc_real_0: 0.09670199453830719 (0.21699146446413714) + | > loss_disc_real_1: 0.2112598568201065 (0.23055492628079194) + | > loss_disc_real_2: 0.1494368016719818 (0.22768039657519415) + | > loss_disc_real_3: 0.228718101978302 (0.23719674363159216) + | > loss_disc_real_4: 0.19139009714126587 (0.23829221138014242) + | > loss_disc_real_5: 0.18362116813659668 (0.23625661704975826) + | > loss_0: 2.6284778118133545 (2.682815535710408) + | > grad_norm_0: tensor(26.8891, device='cuda:0') (tensor(13.5350, device='cuda:0')) + | > loss_gen: 2.174625873565674 (1.9867128775669978) + | > loss_kl: 2.4096357822418213 (2.4395727859093594) + | > loss_feat: 3.2042179107666016 (3.0355417751348934) + | > loss_mel: 25.63088607788086 (25.951763556553768) + | > loss_duration: 1.476050853729248 (1.469200027676729) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.895416259765625 (34.88279122572677) + | > grad_norm_1: tensor(90.6706, device='cuda:0') (tensor(128.2430, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.8289 (0.8256866565117468) + | > loader_time: 0.0122 (0.012063180024807267) + + + --> TIME: 2024-06-25 03:49:41 -- STEP: 129/591 -- GLOBAL_STEP: 3675 + | > loss_disc: 2.6529932022094727 (2.6724334206692006) + | > loss_disc_real_0: 0.2940051853656769 (0.21661681901345878) + | > loss_disc_real_1: 0.21433939039707184 (0.22858281271864278) + | > loss_disc_real_2: 0.23032380640506744 (0.22673878316269364) + | > loss_disc_real_3: 0.2117513120174408 (0.23688533786655397) + | > loss_disc_real_4: 0.26071062684059143 (0.23731579903022262) + | > loss_disc_real_5: 0.2209233194589615 (0.2356440575085869) + | > loss_0: 2.6529932022094727 (2.6724334206692006) + | > grad_norm_0: tensor(16.1766, device='cuda:0') (tensor(13.0514, device='cuda:0')) + | > loss_gen: 2.0839455127716064 (1.989393202833427) + | > loss_kl: 2.360593318939209 (2.4304181771684985) + | > loss_feat: 3.327866554260254 (3.049522133760674) + | > loss_mel: 25.166614532470703 (25.910657897476078) + | > loss_duration: 1.4471982717514038 (1.4713955332142439) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.38621520996094 (34.85138708306836) + | > grad_norm_1: tensor(63.7259, device='cuda:0') (tensor(127.1036, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.8636 (0.8299559585807859) + | > loader_time: 0.012 (0.011966289475906725) + + + --> TIME: 2024-06-25 03:50:03 -- STEP: 154/591 -- GLOBAL_STEP: 3700 + | > loss_disc: 2.7483160495758057 (2.6678144343487626) + | > loss_disc_real_0: 0.22903862595558167 (0.2159009040272855) + | > loss_disc_real_1: 0.2453826367855072 (0.22768167106362133) + | > loss_disc_real_2: 0.24891731142997742 (0.2261250103255371) + | > loss_disc_real_3: 0.29074838757514954 (0.23646693986344647) + | > loss_disc_real_4: 0.2582883834838867 (0.23683534223924985) + | > loss_disc_real_5: 0.23535647988319397 (0.23594716500926327) + | > loss_0: 2.7483160495758057 (2.6678144343487626) + | > grad_norm_0: tensor(7.1256, device='cuda:0') (tensor(13.2755, device='cuda:0')) + | > loss_gen: 1.9144439697265625 (1.9901153677469725) + | > loss_kl: 2.36388897895813 (2.417314908721231) + | > loss_feat: 2.971958875656128 (3.0612845684026726) + | > loss_mel: 24.94622039794922 (25.842203474664068) + | > loss_duration: 1.4496599435806274 (1.469939262061924) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.64617156982422 (34.78085775499219) + | > grad_norm_1: tensor(57.6633, device='cuda:0') (tensor(127.3475, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.8528 (0.8339000955804602) + | > loader_time: 0.0117 (0.011919099015074888) + + + --> TIME: 2024-06-25 03:50:25 -- STEP: 179/591 -- GLOBAL_STEP: 3725 + | > loss_disc: 2.6125645637512207 (2.6647657735387718) + | > loss_disc_real_0: 0.19351691007614136 (0.21574881244138633) + | > loss_disc_real_1: 0.17085622251033783 (0.2261840641998046) + | > loss_disc_real_2: 0.13247635960578918 (0.22514249983779544) + | > loss_disc_real_3: 0.1701652705669403 (0.23595297128461593) + | > loss_disc_real_4: 0.20833221077919006 (0.23677534888909516) + | > loss_disc_real_5: 0.2725401222705841 (0.23636041406812613) + | > loss_0: 2.6125645637512207 (2.6647657735387718) + | > grad_norm_0: tensor(13.1233, device='cuda:0') (tensor(13.3126, device='cuda:0')) + | > loss_gen: 1.958985447883606 (1.9900761729512135) + | > loss_kl: 2.417725086212158 (2.417015151604594) + | > loss_feat: 2.997063398361206 (3.062866438700498) + | > loss_mel: 25.10823631286621 (25.82326497978339) + | > loss_duration: 1.4875733852386475 (1.4682568784532606) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.969581604003906 (34.761479724052876) + | > grad_norm_1: tensor(139.9753, device='cuda:0') (tensor(128.5829, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.876 (0.8385536670684814) + | > loader_time: 0.0114 (0.011928100159714341) + + + --> TIME: 2024-06-25 03:50:48 -- STEP: 204/591 -- GLOBAL_STEP: 3750 + | > loss_disc: 2.548503875732422 (2.6630391595410368) + | > loss_disc_real_0: 0.20669639110565186 (0.2153351793394369) + | > loss_disc_real_1: 0.19403178989887238 (0.2253977264843735) + | > loss_disc_real_2: 0.19380544126033783 (0.2253042496010369) + | > loss_disc_real_3: 0.23460176587104797 (0.23649022436025097) + | > loss_disc_real_4: 0.23237890005111694 (0.23646231338966126) + | > loss_disc_real_5: 0.2081366926431656 (0.2356977015733719) + | > loss_0: 2.548503875732422 (2.6630391595410368) + | > grad_norm_0: tensor(4.7043, device='cuda:0') (tensor(13.2081, device='cuda:0')) + | > loss_gen: 1.8920607566833496 (1.989239562960232) + | > loss_kl: 2.385396957397461 (2.410793548705531) + | > loss_feat: 3.121817111968994 (3.065209791940799) + | > loss_mel: 25.527498245239258 (25.805672533371872) + | > loss_duration: 1.4372369050979614 (1.4673034215674687) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.364009857177734 (34.73821886847999) + | > grad_norm_1: tensor(120.5413, device='cuda:0') (tensor(127.1647, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.8663 (0.8433295196177912) + | > loader_time: 0.0139 (0.012059198875053256) + + + --> TIME: 2024-06-25 03:51:10 -- STEP: 229/591 -- GLOBAL_STEP: 3775 + | > loss_disc: 2.628035306930542 (2.663199323754123) + | > loss_disc_real_0: 0.2549562454223633 (0.21565956488967464) + | > loss_disc_real_1: 0.2557829022407532 (0.22529479353708992) + | > loss_disc_real_2: 0.20409464836120605 (0.22482699351019214) + | > loss_disc_real_3: 0.25899630784988403 (0.236195837826708) + | > loss_disc_real_4: 0.25942349433898926 (0.23607914598748153) + | > loss_disc_real_5: 0.18342289328575134 (0.23581092899022665) + | > loss_0: 2.628035306930542 (2.663199323754123) + | > grad_norm_0: tensor(10.2357, device='cuda:0') (tensor(13.3442, device='cuda:0')) + | > loss_gen: 1.8397080898284912 (1.9891616444400304) + | > loss_kl: 2.357017993927002 (2.4040104801477833) + | > loss_feat: 3.0127124786376953 (3.06473420697008) + | > loss_mel: 25.725873947143555 (25.79488101276248) + | > loss_duration: 1.501044511795044 (1.467035809458604) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.43635559082031 (34.719823112654375) + | > grad_norm_1: tensor(202.3558, device='cuda:0') (tensor(129.7529, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.8784 (0.8471531586876082) + | > loader_time: 0.0152 (0.012291444961681115) + + + --> TIME: 2024-06-25 03:51:33 -- STEP: 254/591 -- GLOBAL_STEP: 3800 + | > loss_disc: 2.650646209716797 (2.662330262304291) + | > loss_disc_real_0: 0.1375017613172531 (0.2147208295586541) + | > loss_disc_real_1: 0.19634795188903809 (0.22522630401718335) + | > loss_disc_real_2: 0.28366726636886597 (0.2249777291234084) + | > loss_disc_real_3: 0.20055072009563446 (0.235973856697871) + | > loss_disc_real_4: 0.2188080996274948 (0.23601149392174922) + | > loss_disc_real_5: 0.2027721405029297 (0.2357659606248375) + | > loss_0: 2.650646209716797 (2.662330262304291) + | > grad_norm_0: tensor(18.4355, device='cuda:0') (tensor(12.9190, device='cuda:0')) + | > loss_gen: 2.096334457397461 (1.9894076829820166) + | > loss_kl: 2.5243611335754395 (2.399181507703827) + | > loss_feat: 3.1634273529052734 (3.064380545315778) + | > loss_mel: 26.15045166015625 (25.783670425415043) + | > loss_duration: 1.4179072380065918 (1.4670603749320268) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 35.352481842041016 (34.70370047862136) + | > grad_norm_1: tensor(65.2149, device='cuda:0') (tensor(125.8851, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.9018 (0.8525632281941692) + | > loader_time: 0.0158 (0.012525980866800143) + + + --> TIME: 2024-06-25 03:51:56 -- STEP: 279/591 -- GLOBAL_STEP: 3825 + | > loss_disc: 2.5407769680023193 (2.66437784061637) + | > loss_disc_real_0: 0.33180007338523865 (0.21539048922638737) + | > loss_disc_real_1: 0.19752846658229828 (0.22522691410288587) + | > loss_disc_real_2: 0.2316632866859436 (0.22534696504099822) + | > loss_disc_real_3: 0.19912900030612946 (0.23601810863795675) + | > loss_disc_real_4: 0.23184242844581604 (0.23558362111395834) + | > loss_disc_real_5: 0.2694234848022461 (0.23627192673930986) + | > loss_0: 2.5407769680023193 (2.66437784061637) + | > grad_norm_0: tensor(22.4247, device='cuda:0') (tensor(13.1822, device='cuda:0')) + | > loss_gen: 1.861966848373413 (1.988512956113371) + | > loss_kl: 2.40868878364563 (2.3951169085758997) + | > loss_feat: 3.2024824619293213 (3.0599605763684874) + | > loss_mel: 25.07661247253418 (25.757485775964668) + | > loss_duration: 1.4504178762435913 (1.467306743385972) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.00016784667969 (34.668382924944694) + | > grad_norm_1: tensor(88.6060, device='cuda:0') (tensor(123.6218, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.9239 (0.8568116892195944) + | > loader_time: 0.0138 (0.012762654212213332) + + + --> TIME: 2024-06-25 03:52:19 -- STEP: 304/591 -- GLOBAL_STEP: 3850 + | > loss_disc: 2.606889247894287 (2.663458879841001) + | > loss_disc_real_0: 0.24715091288089752 (0.21550215915531704) + | > loss_disc_real_1: 0.19250451028347015 (0.2247567234168711) + | > loss_disc_real_2: 0.18113970756530762 (0.22490541526655616) + | > loss_disc_real_3: 0.28190818428993225 (0.23596735131975852) + | > loss_disc_real_4: 0.21829286217689514 (0.23540762419763364) + | > loss_disc_real_5: 0.26240766048431396 (0.23601069634682253) + | > loss_0: 2.606889247894287 (2.663458879841001) + | > grad_norm_0: tensor(7.7140, device='cuda:0') (tensor(12.8959, device='cuda:0')) + | > loss_gen: 2.0157551765441895 (1.9881221118726229) + | > loss_kl: 2.340366840362549 (2.3905569644350764) + | > loss_feat: 3.1371841430664062 (3.0595565508854996) + | > loss_mel: 25.784854888916016 (25.735721079926755) + | > loss_duration: 1.511021375656128 (1.467965601698349) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.789180755615234 (34.64192227313392) + | > grad_norm_1: tensor(128.0447, device='cuda:0') (tensor(124.4731, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.9275 (0.8619889373842042) + | > loader_time: 0.0137 (0.012982942556080065) + + + --> TIME: 2024-06-25 03:52:43 -- STEP: 329/591 -- GLOBAL_STEP: 3875 + | > loss_disc: 2.732412338256836 (2.664012285713729) + | > loss_disc_real_0: 0.1574765294790268 (0.2155390406723805) + | > loss_disc_real_1: 0.22991929948329926 (0.22491446354831238) + | > loss_disc_real_2: 0.20993450284004211 (0.22476245415337542) + | > loss_disc_real_3: 0.26433515548706055 (0.23602859761939585) + | > loss_disc_real_4: 0.27013227343559265 (0.23565543499579908) + | > loss_disc_real_5: 0.289618581533432 (0.23613897323789568) + | > loss_0: 2.732412338256836 (2.664012285713729) + | > grad_norm_0: tensor(12.7002, device='cuda:0') (tensor(12.8460, device='cuda:0')) + | > loss_gen: 1.9494891166687012 (1.9880876903475961) + | > loss_kl: 2.407381534576416 (2.384921671047037) + | > loss_feat: 3.025054454803467 (3.0586094870755716) + | > loss_mel: 25.134706497192383 (25.720753829167617) + | > loss_duration: 1.512247085571289 (1.468592742656141) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.02887725830078 (34.62096533731849) + | > grad_norm_1: tensor(95.8390, device='cuda:0') (tensor(125.3823, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.9194 (0.8668929573975077) + | > loader_time: 0.0165 (0.0131986344114263) + + + --> TIME: 2024-06-25 03:53:07 -- STEP: 354/591 -- GLOBAL_STEP: 3900 + | > loss_disc: 2.730290651321411 (2.663203822017388) + | > loss_disc_real_0: 0.34086447954177856 (0.21605416795628218) + | > loss_disc_real_1: 0.21024926006793976 (0.22446230251741944) + | > loss_disc_real_2: 0.30703261494636536 (0.22463324019885333) + | > loss_disc_real_3: 0.20428165793418884 (0.23590419243621288) + | > loss_disc_real_4: 0.2827344238758087 (0.23559258697228244) + | > loss_disc_real_5: 0.2797297239303589 (0.23596149257852533) + | > loss_0: 2.730290651321411 (2.663203822017388) + | > grad_norm_0: tensor(22.8258, device='cuda:0') (tensor(12.8686, device='cuda:0')) + | > loss_gen: 1.6310486793518066 (1.9867622805180523) + | > loss_kl: 2.33648419380188 (2.3793956378085457) + | > loss_feat: 3.0159575939178467 (3.055425968547323) + | > loss_mel: 25.644420623779297 (25.702335276846167) + | > loss_duration: 1.490241527557373 (1.469744381258045) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.1181526184082 (34.59366345809678) + | > grad_norm_1: tensor(206.1032, device='cuda:0') (tensor(127.8278, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.9604 (0.8724174533186663) + | > loader_time: 0.0176 (0.013394549741583356) + + + --> TIME: 2024-06-25 03:53:32 -- STEP: 379/591 -- GLOBAL_STEP: 3925 + | > loss_disc: 2.5207901000976562 (2.663118096328976) + | > loss_disc_real_0: 0.16383853554725647 (0.21575659851600124) + | > loss_disc_real_1: 0.2179757058620453 (0.2244094016526808) + | > loss_disc_real_2: 0.18331313133239746 (0.22433955249890175) + | > loss_disc_real_3: 0.2043513059616089 (0.23567183563451338) + | > loss_disc_real_4: 0.1984759420156479 (0.2353836621961367) + | > loss_disc_real_5: 0.24138690531253815 (0.23604602946612324) + | > loss_0: 2.5207901000976562 (2.663118096328976) + | > grad_norm_0: tensor(9.9362, device='cuda:0') (tensor(12.7744, device='cuda:0')) + | > loss_gen: 2.1598587036132812 (1.9863999358582307) + | > loss_kl: 2.3095004558563232 (2.376597946111634) + | > loss_feat: 3.3668594360351562 (3.052208745070684) + | > loss_mel: 25.1009521484375 (25.67947958893387) + | > loss_duration: 1.48677659034729 (1.4712472893002813) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.4239501953125 (34.565933398646855) + | > grad_norm_1: tensor(171.1062, device='cuda:0') (tensor(130.3523, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.9599 (0.877833673381554) + | > loader_time: 0.0182 (0.01360315509081516) + + + --> TIME: 2024-06-25 03:53:57 -- STEP: 404/591 -- GLOBAL_STEP: 3950 + | > loss_disc: 2.630053997039795 (2.6638777173391652) + | > loss_disc_real_0: 0.19983315467834473 (0.2158630296477292) + | > loss_disc_real_1: 0.18978668749332428 (0.224262628446121) + | > loss_disc_real_2: 0.15247103571891785 (0.22419299570714482) + | > loss_disc_real_3: 0.2085956484079361 (0.23573748221491822) + | > loss_disc_real_4: 0.25139740109443665 (0.23559477338844007) + | > loss_disc_real_5: 0.18822485208511353 (0.23598179571551853) + | > loss_0: 2.630053997039795 (2.6638777173391652) + | > grad_norm_0: tensor(7.1375, device='cuda:0') (tensor(12.8358, device='cuda:0')) + | > loss_gen: 2.045689582824707 (1.9846960288463253) + | > loss_kl: 2.3919107913970947 (2.3744262775572236) + | > loss_feat: 3.102128028869629 (3.0464796347193177) + | > loss_mel: 25.04349136352539 (25.65730339937872) + | > loss_duration: 1.46522855758667 (1.472154746846399) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.04844665527344 (34.53505998554796) + | > grad_norm_1: tensor(162.0347, device='cuda:0') (tensor(129.5523, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.9869 (0.8836813067445664) + | > loader_time: 0.0172 (0.01381231947700576) + + + --> TIME: 2024-06-25 03:54:22 -- STEP: 429/591 -- GLOBAL_STEP: 3975 + | > loss_disc: 2.6397948265075684 (2.6635126156128917) + | > loss_disc_real_0: 0.1965709626674652 (0.21585034648507903) + | > loss_disc_real_1: 0.2699747085571289 (0.22422688817366573) + | > loss_disc_real_2: 0.2566206753253937 (0.22398830132259356) + | > loss_disc_real_3: 0.2382500171661377 (0.2358736493440219) + | > loss_disc_real_4: 0.2766669988632202 (0.2355900662831771) + | > loss_disc_real_5: 0.20355825126171112 (0.23575904454801466) + | > loss_0: 2.6397948265075684 (2.6635126156128917) + | > grad_norm_0: tensor(5.1769, device='cuda:0') (tensor(12.7787, device='cuda:0')) + | > loss_gen: 2.1303939819335938 (1.98496841161679) + | > loss_kl: 2.3738083839416504 (2.371780584464264) + | > loss_feat: 2.9262962341308594 (3.0448192255202406) + | > loss_mel: 25.919050216674805 (25.627114058096655) + | > loss_duration: 1.489988088607788 (1.4727033390587598) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.839534759521484 (34.50138553539357) + | > grad_norm_1: tensor(108.8480, device='cuda:0') (tensor(129.0184, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.9761 (0.889942969475593) + | > loader_time: 0.0172 (0.014031969306074378) + + + --> TIME: 2024-06-25 03:54:48 -- STEP: 454/591 -- GLOBAL_STEP: 4000 + | > loss_disc: 2.6245105266571045 (2.66342175321957) + | > loss_disc_real_0: 0.2626006305217743 (0.2158854271773462) + | > loss_disc_real_1: 0.201484814286232 (0.22398364255081712) + | > loss_disc_real_2: 0.1886810064315796 (0.22391910337338364) + | > loss_disc_real_3: 0.19584375619888306 (0.2358510835908583) + | > loss_disc_real_4: 0.20095157623291016 (0.23550475595543563) + | > loss_disc_real_5: 0.23029281198978424 (0.2357982474211029) + | > loss_0: 2.6245105266571045 (2.66342175321957) + | > grad_norm_0: tensor(13.0710, device='cuda:0') (tensor(12.6978, device='cuda:0')) + | > loss_gen: 2.2006680965423584 (1.983806089682726) + | > loss_kl: 2.4490339756011963 (2.369094038849887) + | > loss_feat: 3.065105676651001 (3.0383392866487524) + | > loss_mel: 24.55263900756836 (25.595688542605505) + | > loss_duration: 1.4979033470153809 (1.4735224814141914) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.76534652709961 (34.460450353076304) + | > grad_norm_1: tensor(64.2797, device='cuda:0') (tensor(127.1359, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 0.9975 (0.8966027986635723) + | > loader_time: 0.024 (0.014295704564334013) + + + --> TIME: 2024-06-25 03:55:14 -- STEP: 479/591 -- GLOBAL_STEP: 4025 + | > loss_disc: 2.6720404624938965 (2.6645608326587387) + | > loss_disc_real_0: 0.23440393805503845 (0.21589316385710663) + | > loss_disc_real_1: 0.3296770453453064 (0.22409720401549885) + | > loss_disc_real_2: 0.2500693202018738 (0.22422123006651945) + | > loss_disc_real_3: 0.2523300349712372 (0.23616470909815493) + | > loss_disc_real_4: 0.18601259589195251 (0.23548656459392237) + | > loss_disc_real_5: 0.2276242971420288 (0.23576275839437275) + | > loss_0: 2.6720404624938965 (2.6645608326587387) + | > grad_norm_0: tensor(8.8453, device='cuda:0') (tensor(12.7709, device='cuda:0')) + | > loss_gen: 1.9045906066894531 (1.9824033408174933) + | > loss_kl: 2.2957093715667725 (2.366804667454922) + | > loss_feat: 2.9720983505249023 (3.0319719827249783) + | > loss_mel: 24.871402740478516 (25.571470101342577) + | > loss_duration: 1.5632967948913574 (1.4747873522294603) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.60709762573242 (34.427437362193096) + | > grad_norm_1: tensor(203.7946, device='cuda:0') (tensor(127.6786, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 1.0027 (0.9027924224081022) + | > loader_time: 0.0201 (0.014571235672666036) + + + --> TIME: 2024-06-25 03:55:41 -- STEP: 504/591 -- GLOBAL_STEP: 4050 + | > loss_disc: 2.7050228118896484 (2.6644376374426333) + | > loss_disc_real_0: 0.20232923328876495 (0.21569568839752012) + | > loss_disc_real_1: 0.2120012491941452 (0.22396037777856226) + | > loss_disc_real_2: 0.2086893767118454 (0.22430250756738204) + | > loss_disc_real_3: 0.2157890349626541 (0.23609442423496926) + | > loss_disc_real_4: 0.2592369318008423 (0.23563954180904798) + | > loss_disc_real_5: 0.25171202421188354 (0.2358272046974254) + | > loss_0: 2.7050228118896484 (2.6644376374426333) + | > grad_norm_0: tensor(4.8366, device='cuda:0') (tensor(12.7353, device='cuda:0')) + | > loss_gen: 1.9686871767044067 (1.9821091526084476) + | > loss_kl: 2.301439046859741 (2.3648841357420376) + | > loss_feat: 2.644181489944458 (3.0311910680362137) + | > loss_mel: 25.68547821044922 (25.546612777407216) + | > loss_duration: 1.515642523765564 (1.4759879332213188) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.11543273925781 (34.40078498446755) + | > grad_norm_1: tensor(189.8631, device='cuda:0') (tensor(128.9650, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 1.0893 (0.910880956384871) + | > loader_time: 0.0207 (0.014842985641388666) + + + --> TIME: 2024-06-25 03:56:10 -- STEP: 529/591 -- GLOBAL_STEP: 4075 + | > loss_disc: 2.602872133255005 (2.6638430931617716) + | > loss_disc_real_0: 0.21247772872447968 (0.21554772157243157) + | > loss_disc_real_1: 0.21724794805049896 (0.22391651246858674) + | > loss_disc_real_2: 0.21157673001289368 (0.2242779027395979) + | > loss_disc_real_3: 0.23814867436885834 (0.2360656492144489) + | > loss_disc_real_4: 0.23887719213962555 (0.23555317028145709) + | > loss_disc_real_5: 0.2551068365573883 (0.23573408756116387) + | > loss_0: 2.602872133255005 (2.6638430931617716) + | > grad_norm_0: tensor(3.9010, device='cuda:0') (tensor(12.6583, device='cuda:0')) + | > loss_gen: 2.0924899578094482 (1.9815716344602616) + | > loss_kl: 2.48504376411438 (2.3629644877067824) + | > loss_feat: 3.054691791534424 (3.029557667067009) + | > loss_mel: 25.533647537231445 (25.52355410426236) + | > loss_duration: 1.5051393508911133 (1.4773623150102595) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 34.67101287841797 (34.375010146085195) + | > grad_norm_1: tensor(115.6383, device='cuda:0') (tensor(129.1708, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 1.2368 (0.9207384356478889) + | > loader_time: 0.0223 (0.015128041034835497) + + + --> TIME: 2024-06-25 03:56:40 -- STEP: 554/591 -- GLOBAL_STEP: 4100 + | > loss_disc: 2.626516103744507 (2.665133776647519) + | > loss_disc_real_0: 0.1654849797487259 (0.21557542660176107) + | > loss_disc_real_1: 0.18798106908798218 (0.22383685705894163) + | > loss_disc_real_2: 0.24442334473133087 (0.22435793464848716) + | > loss_disc_real_3: 0.2543337643146515 (0.23605778859087706) + | > loss_disc_real_4: 0.20273716747760773 (0.23572348625758924) + | > loss_disc_real_5: 0.19722479581832886 (0.23586561353305616) + | > loss_0: 2.626516103744507 (2.665133776647519) + | > grad_norm_0: tensor(10.0650, device='cuda:0') (tensor(12.6485, device='cuda:0')) + | > loss_gen: 2.046574831008911 (1.979756237152251) + | > loss_kl: 2.3367319107055664 (2.361366444976752) + | > loss_feat: 3.0595717430114746 (3.022100418052947) + | > loss_mel: 25.243379592895508 (25.487648526684048) + | > loss_duration: 1.5096678733825684 (1.4788003460594903) + | > amp_scaler: 1024.0 (514.7725631768953) + | > loss_1: 34.1959228515625 (34.32967191482712) + | > grad_norm_1: tensor(68.7396, device='cuda:0') (tensor(128.5726, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 1.1585 (0.93279003795734) + | > loader_time: 0.0252 (0.015475754273066883) + + + --> TIME: 2024-06-25 03:57:11 -- STEP: 579/591 -- GLOBAL_STEP: 4125 + | > loss_disc: 2.8291053771972656 (2.6661181919315315) + | > loss_disc_real_0: 0.2234596312046051 (0.21558633588241594) + | > loss_disc_real_1: 0.25595715641975403 (0.22405171106104602) + | > loss_disc_real_2: 0.25699084997177124 (0.22456336161684712) + | > loss_disc_real_3: 0.2812751233577728 (0.23630666920661103) + | > loss_disc_real_4: 0.2550293505191803 (0.23581238801306392) + | > loss_disc_real_5: 0.2807907164096832 (0.23599699013488487) + | > loss_0: 2.8291053771972656 (2.6661181919315315) + | > grad_norm_0: tensor(4.7577, device='cuda:0') (tensor(12.6059, device='cuda:0')) + | > loss_gen: 1.687657356262207 (1.9792664256944146) + | > loss_kl: 2.2955355644226074 (2.35824585496247) + | > loss_feat: 2.635573625564575 (3.0161709653692936) + | > loss_mel: 25.26297378540039 (25.450625058900517) + | > loss_duration: 1.5140283107757568 (1.480258598846476) + | > amp_scaler: 1024.0 (536.7599309153715) + | > loss_1: 33.39576721191406 (34.28456684303617) + | > grad_norm_1: tensor(184.7568, device='cuda:0') (tensor(127.7144, device='cuda:0')) + | > current_lr_0: 0.0001998500468671882 + | > current_lr_1: 0.0001998500468671882 + | > step_time: 1.3187 (0.9454406621336735) + | > loader_time: 0.0248 (0.015836520927969455) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003691100059671605 (+8.33247570281334e-05) + | > avg_loss_disc: 2.6834928837228333 (+0.08607705603254656) + | > avg_loss_disc_real_0: 0.16061531054846784 (+0.0032869869090141612) + | > avg_loss_disc_real_1: 0.1881321025021533 (+0.01829527508705217) + | > avg_loss_disc_real_2: 0.1912638858911839 (-0.002541994477840148) + | > avg_loss_disc_real_3: 0.22225296433935773 (-0.009813793479128086) + | > avg_loss_disc_real_4: 0.21775069674278827 (-0.021940419648556003) + | > avg_loss_disc_real_5: 0.22718993400005585 (+0.005309546247441732) + | > avg_loss_0: 2.6834928837228333 (+0.08607705603254656) + | > avg_loss_gen: 1.7028755456843274 (-0.07507577601899507) + | > avg_loss_kl: 2.360800377866055 (+0.07294146304435056) + | > avg_loss_feat: 2.9721476321524762 (+0.05107647307375629) + | > avg_loss_mel: 25.04776354038969 (-0.7677825765406858) + | > avg_loss_duration: 1.4567060318398983 (-0.02018591698179839) + | > avg_loss_1: 33.54029310510514 (-0.7390265363327941) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_4137.pth + + > EPOCH: 8/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 03:57:38)  + + --> TIME: 2024-06-25 03:57:51 -- STEP: 13/591 -- GLOBAL_STEP: 4150 + | > loss_disc: 2.706383466720581 (2.6651250215677114) + | > loss_disc_real_0: 0.2969590425491333 (0.21844141070659345) + | > loss_disc_real_1: 0.24018466472625732 (0.22494503397207993) + | > loss_disc_real_2: 0.32168999314308167 (0.23695785265702468) + | > loss_disc_real_3: 0.21695785224437714 (0.23981826465863448) + | > loss_disc_real_4: 0.2950606048107147 (0.23841689985532027) + | > loss_disc_real_5: 0.19807234406471252 (0.23861502913328317) + | > loss_0: 2.706383466720581 (2.6651250215677114) + | > grad_norm_0: tensor(14.5505, device='cuda:0') (tensor(12.1035, device='cuda:0')) + | > loss_gen: 1.8206987380981445 (2.006307629438547) + | > loss_kl: 2.6382479667663574 (2.548395175200242) + | > loss_feat: 2.988969326019287 (3.2187993159660926) + | > loss_mel: 24.672046661376953 (25.452664301945614) + | > loss_duration: 1.4349544048309326 (1.4136853768275335) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 33.55491638183594 (34.63985149676983) + | > grad_norm_1: tensor(116.8258, device='cuda:0') (tensor(121.0367, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.7774 (0.7768703974210299) + | > loader_time: 0.0086 (0.008771107747004582) + + + --> TIME: 2024-06-25 03:58:11 -- STEP: 38/591 -- GLOBAL_STEP: 4175 + | > loss_disc: 2.6896378993988037 (2.673797550954317) + | > loss_disc_real_0: 0.1694813072681427 (0.21782890079837097) + | > loss_disc_real_1: 0.2060440331697464 (0.22268517158533396) + | > loss_disc_real_2: 0.17910529673099518 (0.22767740566479533) + | > loss_disc_real_3: 0.2762052118778229 (0.24129792734196312) + | > loss_disc_real_4: 0.21692527830600739 (0.23539412374559202) + | > loss_disc_real_5: 0.210801362991333 (0.23725722612519012) + | > loss_0: 2.6896378993988037 (2.673797550954317) + | > grad_norm_0: tensor(8.3588, device='cuda:0') (tensor(12.9205, device='cuda:0')) + | > loss_gen: 1.9399516582489014 (1.9842410777744495) + | > loss_kl: 2.4607162475585938 (2.4962504474740275) + | > loss_feat: 3.2328827381134033 (3.1253229128687003) + | > loss_mel: 25.340646743774414 (25.066277704740823) + | > loss_duration: 1.4457933902740479 (1.4265721032493992) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 34.41999053955078 (34.09866383201197) + | > grad_norm_1: tensor(165.7538, device='cuda:0') (tensor(115.6252, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.7982 (0.7858565983019378) + | > loader_time: 0.01 (0.00938372862966437) + + + --> TIME: 2024-06-25 03:58:31 -- STEP: 63/591 -- GLOBAL_STEP: 4200 + | > loss_disc: 2.689426898956299 (2.6730084078652516) + | > loss_disc_real_0: 0.15589453279972076 (0.21838974574255565) + | > loss_disc_real_1: 0.20179535448551178 (0.22614966545786178) + | > loss_disc_real_2: 0.25613242387771606 (0.22653059448514665) + | > loss_disc_real_3: 0.2739948332309723 (0.23857085477738155) + | > loss_disc_real_4: 0.23719783127307892 (0.23584283249718802) + | > loss_disc_real_5: 0.27702227234840393 (0.23726780996436164) + | > loss_0: 2.689426898956299 (2.6730084078652516) + | > grad_norm_0: tensor(10.8631, device='cuda:0') (tensor(12.8678, device='cuda:0')) + | > loss_gen: 1.973509669303894 (1.9903650075670272) + | > loss_kl: 2.401853084564209 (2.466123550657242) + | > loss_feat: 3.1838202476501465 (3.1309039138612293) + | > loss_mel: 25.916894912719727 (25.118662607102166) + | > loss_duration: 1.4860913753509521 (1.4402754893378604) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 34.9621696472168 (34.14633027334061) + | > grad_norm_1: tensor(185.7279, device='cuda:0') (tensor(130.5333, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.8061 (0.7955194503541978) + | > loader_time: 0.0106 (0.009786231177193782) + + + --> TIME: 2024-06-25 03:58:52 -- STEP: 88/591 -- GLOBAL_STEP: 4225 + | > loss_disc: 2.5406839847564697 (2.667531847953796) + | > loss_disc_real_0: 0.16006115078926086 (0.21637818572873418) + | > loss_disc_real_1: 0.1784021258354187 (0.22628268582577055) + | > loss_disc_real_2: 0.15623755753040314 (0.22513731907714496) + | > loss_disc_real_3: 0.17154501378536224 (0.23593519374050878) + | > loss_disc_real_4: 0.19737304747104645 (0.23577637394720857) + | > loss_disc_real_5: 0.20001186430454254 (0.23585564660077746) + | > loss_0: 2.5406839847564697 (2.667531847953796) + | > grad_norm_0: tensor(9.6464, device='cuda:0') (tensor(11.9057, device='cuda:0')) + | > loss_gen: 2.0456907749176025 (1.9854219637133859) + | > loss_kl: 2.2875988483428955 (2.4383869279514667) + | > loss_feat: 3.435696840286255 (3.1280638277530666) + | > loss_mel: 25.553817749023438 (25.17516199025241) + | > loss_duration: 1.5217827558517456 (1.4491300460967151) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 34.84458541870117 (34.1761644970287) + | > grad_norm_1: tensor(177.5706, device='cuda:0') (tensor(134.8137, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.8401 (0.8040774627165362) + | > loader_time: 0.0113 (0.010121193799105558) + + + --> TIME: 2024-06-25 03:59:14 -- STEP: 113/591 -- GLOBAL_STEP: 4250 + | > loss_disc: 2.61281156539917 (2.663899073558571) + | > loss_disc_real_0: 0.16587121784687042 (0.21640302550739945) + | > loss_disc_real_1: 0.2129320353269577 (0.22572168773781937) + | > loss_disc_real_2: 0.2743208706378937 (0.22564608742178013) + | > loss_disc_real_3: 0.2069985568523407 (0.23578229612481277) + | > loss_disc_real_4: 0.2770756781101227 (0.2356471127904622) + | > loss_disc_real_5: 0.30810070037841797 (0.23635294321364006) + | > loss_0: 2.61281156539917 (2.663899073558571) + | > grad_norm_0: tensor(11.2295, device='cuda:0') (tensor(12.0998, device='cuda:0')) + | > loss_gen: 1.8224058151245117 (1.9906937358653651) + | > loss_kl: 2.4855644702911377 (2.4250779637193256) + | > loss_feat: 3.3095805644989014 (3.1513383915994013) + | > loss_mel: 24.79023551940918 (25.177897292955787) + | > loss_duration: 1.447129249572754 (1.4537451530979797) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 33.854915618896484 (34.1987523441821) + | > grad_norm_1: tensor(192.9334, device='cuda:0') (tensor(134.4210, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.8324 (0.8115350060758338) + | > loader_time: 0.0107 (0.01050582185255743) + + + --> TIME: 2024-06-25 03:59:35 -- STEP: 138/591 -- GLOBAL_STEP: 4275 + | > loss_disc: 2.5980916023254395 (2.6613950850307075) + | > loss_disc_real_0: 0.23006057739257812 (0.2159821854877299) + | > loss_disc_real_1: 0.19222207367420197 (0.2249061283857926) + | > loss_disc_real_2: 0.21856625378131866 (0.22483023558405862) + | > loss_disc_real_3: 0.22343239188194275 (0.2357119366094686) + | > loss_disc_real_4: 0.2352399230003357 (0.23565242785042612) + | > loss_disc_real_5: 0.2313353568315506 (0.2359229099491368) + | > loss_0: 2.5980916023254395 (2.6613950850307075) + | > grad_norm_0: tensor(8.3750, device='cuda:0') (tensor(11.6779, device='cuda:0')) + | > loss_gen: 1.8140777349472046 (1.9913098414738972) + | > loss_kl: 2.2656280994415283 (2.418251875518025) + | > loss_feat: 3.1282365322113037 (3.15172589343527) + | > loss_mel: 25.441694259643555 (25.19220840067103) + | > loss_duration: 1.4666743278503418 (1.4549476556155994) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 34.11631393432617 (34.20844357255575) + | > grad_norm_1: tensor(220.0603, device='cuda:0') (tensor(134.8532, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.838 (0.817658165226812) + | > loader_time: 0.011 (0.010765898054924564) + + + --> TIME: 2024-06-25 03:59:57 -- STEP: 163/591 -- GLOBAL_STEP: 4300 + | > loss_disc: 2.6013998985290527 (2.6586009593097706) + | > loss_disc_real_0: 0.20980007946491241 (0.21583823626575294) + | > loss_disc_real_1: 0.19601695239543915 (0.22383818981106296) + | > loss_disc_real_2: 0.12943334877490997 (0.22383655808820316) + | > loss_disc_real_3: 0.13638892769813538 (0.23479900257719075) + | > loss_disc_real_4: 0.20456470549106598 (0.2348675218645049) + | > loss_disc_real_5: 0.18219739198684692 (0.23575035478074127) + | > loss_0: 2.6013998985290527 (2.6586009593097706) + | > grad_norm_0: tensor(10.9613, device='cuda:0') (tensor(11.9155, device='cuda:0')) + | > loss_gen: 2.1719560623168945 (1.9918252697751566) + | > loss_kl: 2.3477625846862793 (2.412873332485831) + | > loss_feat: 3.4011142253875732 (3.1528278230889435) + | > loss_mel: 25.49055290222168 (25.165405320243607) + | > loss_duration: 1.4576395750045776 (1.4533876224529527) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 34.869022369384766 (34.17631926273275) + | > grad_norm_1: tensor(100.5721, device='cuda:0') (tensor(132.6846, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.8547 (0.8235660316022627) + | > loader_time: 0.0112 (0.010943428870359078) + + + --> TIME: 2024-06-25 04:00:19 -- STEP: 188/591 -- GLOBAL_STEP: 4325 + | > loss_disc: 2.526081085205078 (2.657726570646814) + | > loss_disc_real_0: 0.12126569449901581 (0.21488211624958414) + | > loss_disc_real_1: 0.21294625103473663 (0.22399007560724907) + | > loss_disc_real_2: 0.23415754735469818 (0.22465394532426874) + | > loss_disc_real_3: 0.21326011419296265 (0.23517493079317378) + | > loss_disc_real_4: 0.22453497350215912 (0.2349392648072953) + | > loss_disc_real_5: 0.2255069613456726 (0.23584866325588935) + | > loss_0: 2.526081085205078 (2.657726570646814) + | > grad_norm_0: tensor(19.3817, device='cuda:0') (tensor(11.7528, device='cuda:0')) + | > loss_gen: 2.167991876602173 (1.992481861976867) + | > loss_kl: 2.1766648292541504 (2.4113377700460714) + | > loss_feat: 3.39359188079834 (3.150350024091436) + | > loss_mel: 25.22467041015625 (25.157109321431914) + | > loss_duration: 1.4619109630584717 (1.4521363783389971) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 34.42483139038086 (34.163415239212334) + | > grad_norm_1: tensor(84.8995, device='cuda:0') (tensor(133.3828, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.865 (0.8289303462556068) + | > loader_time: 0.0128 (0.011098564939295992) + + + --> TIME: 2024-06-25 04:00:42 -- STEP: 213/591 -- GLOBAL_STEP: 4350 + | > loss_disc: 2.715686798095703 (2.6619709131303533) + | > loss_disc_real_0: 0.19823120534420013 (0.21540034768726904) + | > loss_disc_real_1: 0.23512765765190125 (0.22418296078281225) + | > loss_disc_real_2: 0.2359103113412857 (0.22456126901465404) + | > loss_disc_real_3: 0.3405958116054535 (0.23609360331940538) + | > loss_disc_real_4: 0.22952882945537567 (0.2356464575294038) + | > loss_disc_real_5: 0.21567648649215698 (0.23622842496829413) + | > loss_0: 2.715686798095703 (2.6619709131303533) + | > grad_norm_0: tensor(6.7340, device='cuda:0') (tensor(11.7214, device='cuda:0')) + | > loss_gen: 1.938413143157959 (1.9931643938234713) + | > loss_kl: 2.3545820713043213 (2.4074914018872757) + | > loss_feat: 2.8973588943481445 (3.145255436919664) + | > loss_mel: 24.862131118774414 (25.129474245886293) + | > loss_duration: 1.4473940134048462 (1.4512181897678291) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 33.4998779296875 (34.12660353396419) + | > grad_norm_1: tensor(115.8268, device='cuda:0') (tensor(131.9247, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.8719 (0.8347550848839989) + | > loader_time: 0.012 (0.011323143059099225) + + + --> TIME: 2024-06-25 04:01:04 -- STEP: 238/591 -- GLOBAL_STEP: 4375 + | > loss_disc: 2.7057278156280518 (2.6667297697868673) + | > loss_disc_real_0: 0.16396448016166687 (0.21656607714395562) + | > loss_disc_real_1: 0.23438309133052826 (0.22396613826521306) + | > loss_disc_real_2: 0.18309684097766876 (0.2252432206222991) + | > loss_disc_real_3: 0.18796662986278534 (0.23555182797067306) + | > loss_disc_real_4: 0.23852045834064484 (0.23575513872529277) + | > loss_disc_real_5: 0.2272709161043167 (0.23619384374939092) + | > loss_0: 2.7057278156280518 (2.6667297697868673) + | > grad_norm_0: tensor(11.6261, device='cuda:0') (tensor(11.7843, device='cuda:0')) + | > loss_gen: 2.0103647708892822 (1.9896074693743921) + | > loss_kl: 2.275125026702881 (2.404631237022015) + | > loss_feat: 3.056715965270996 (3.1271676876965686) + | > loss_mel: 24.719331741333008 (25.10559406601081) + | > loss_duration: 1.4154536724090576 (1.4509982351495443) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 33.476993560791016 (34.077998505920924) + | > grad_norm_1: tensor(70.9106, device='cuda:0') (tensor(128.8512, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.8746 (0.8394864797592163) + | > loader_time: 0.0134 (0.011651153324031029) + + + --> TIME: 2024-06-25 04:01:27 -- STEP: 263/591 -- GLOBAL_STEP: 4400 + | > loss_disc: 2.723148822784424 (2.667901403550413) + | > loss_disc_real_0: 0.28787410259246826 (0.21680171054012423) + | > loss_disc_real_1: 0.20319801568984985 (0.2235704658149313) + | > loss_disc_real_2: 0.21345382928848267 (0.22531917560916437) + | > loss_disc_real_3: 0.34095054864883423 (0.23591094523554973) + | > loss_disc_real_4: 0.25547051429748535 (0.2357031879202948) + | > loss_disc_real_5: 0.2300199270248413 (0.23617539200719317) + | > loss_0: 2.723148822784424 (2.667901403550413) + | > grad_norm_0: tensor(17.2756, device='cuda:0') (tensor(11.6838, device='cuda:0')) + | > loss_gen: 1.8134112358093262 (1.9856465166512551) + | > loss_kl: 2.363776445388794 (2.4009158121768968) + | > loss_feat: 3.04852032661438 (3.1128369447396276) + | > loss_mel: 25.430438995361328 (25.08798320030532) + | > loss_duration: 1.4235787391662598 (1.4509623744188611) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 34.07972717285156 (34.038344713218315) + | > grad_norm_1: tensor(62.7600, device='cuda:0') (tensor(130.3535, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.911 (0.8449120566872136) + | > loader_time: 0.0152 (0.011986077058451257) + + + --> TIME: 2024-06-25 04:01:50 -- STEP: 288/591 -- GLOBAL_STEP: 4425 + | > loss_disc: 2.695690631866455 (2.667834508750174) + | > loss_disc_real_0: 0.24593402445316315 (0.21624290386939216) + | > loss_disc_real_1: 0.1711776852607727 (0.22345142061304715) + | > loss_disc_real_2: 0.20469041168689728 (0.22552779265162018) + | > loss_disc_real_3: 0.22180862724781036 (0.23556419265353018) + | > loss_disc_real_4: 0.24338966608047485 (0.2358238632376823) + | > loss_disc_real_5: 0.2671782672405243 (0.2361647821445432) + | > loss_0: 2.695690631866455 (2.667834508750174) + | > grad_norm_0: tensor(7.7371, device='cuda:0') (tensor(11.6185, device='cuda:0')) + | > loss_gen: 1.8053240776062012 (1.9839286067419581) + | > loss_kl: 2.350802183151245 (2.398236469262176) + | > loss_feat: 2.871185779571533 (3.1060630041691986) + | > loss_mel: 25.19211769104004 (25.066627727614517) + | > loss_duration: 1.4945831298828125 (1.4517791958318815) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 33.714012145996094 (34.006634818183045) + | > grad_norm_1: tensor(75.2345, device='cuda:0') (tensor(128.4089, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.8964 (0.8497079668773544) + | > loader_time: 0.0164 (0.012271280917856429) + + + --> TIME: 2024-06-25 04:02:13 -- STEP: 313/591 -- GLOBAL_STEP: 4450 + | > loss_disc: 2.620713710784912 (2.6696281288378527) + | > loss_disc_real_0: 0.26243895292282104 (0.21626841629179902) + | > loss_disc_real_1: 0.20236027240753174 (0.2234471715962925) + | > loss_disc_real_2: 0.2628840506076813 (0.2255749442040349) + | > loss_disc_real_3: 0.2594968378543854 (0.23549716165081) + | > loss_disc_real_4: 0.17109553515911102 (0.23582977213608192) + | > loss_disc_real_5: 0.17808407545089722 (0.23604354857446286) + | > loss_0: 2.620713710784912 (2.6696281288378527) + | > grad_norm_0: tensor(10.9706, device='cuda:0') (tensor(11.6102, device='cuda:0')) + | > loss_gen: 1.9606776237487793 (1.9801621875062156) + | > loss_kl: 2.2687578201293945 (2.397159072918631) + | > loss_feat: 3.057314872741699 (3.0943797746786292) + | > loss_mel: 25.549232482910156 (25.04930946545099) + | > loss_duration: 1.433274507522583 (1.4520970026905928) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 34.269256591796875 (33.97310732271723) + | > grad_norm_1: tensor(85.6165, device='cuda:0') (tensor(127.3137, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.9172 (0.8553673444083705) + | > loader_time: 0.0141 (0.01253448050623885) + + + --> TIME: 2024-06-25 04:02:37 -- STEP: 338/591 -- GLOBAL_STEP: 4475 + | > loss_disc: 2.704871892929077 (2.6706848116316033) + | > loss_disc_real_0: 0.21636147797107697 (0.21622068878020761) + | > loss_disc_real_1: 0.24107173085212708 (0.22336241166443516) + | > loss_disc_real_2: 0.347940593957901 (0.22601635635075484) + | > loss_disc_real_3: 0.19861958920955658 (0.23553641241859402) + | > loss_disc_real_4: 0.26147857308387756 (0.23597980206887398) + | > loss_disc_real_5: 0.2573542892932892 (0.23601246642819523) + | > loss_0: 2.704871892929077 (2.6706848116316033) + | > grad_norm_0: tensor(8.6900, device='cuda:0') (tensor(11.5592, device='cuda:0')) + | > loss_gen: 1.9723457098007202 (1.9786294413036143) + | > loss_kl: 2.367241859436035 (2.3937639188484323) + | > loss_feat: 3.049471378326416 (3.086304915727241) + | > loss_mel: 25.631940841674805 (25.04951856827596) + | > loss_duration: 1.497537612915039 (1.453065450374897) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 34.51853942871094 (33.96128213193994) + | > grad_norm_1: tensor(86.1888, device='cuda:0') (tensor(126.3653, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.9171 (0.8607269807680119) + | > loader_time: 0.0146 (0.012802051369255114) + + + --> TIME: 2024-06-25 04:03:01 -- STEP: 363/591 -- GLOBAL_STEP: 4500 + | > loss_disc: 2.7470543384552 (2.673948842960286) + | > loss_disc_real_0: 0.24042919278144836 (0.21656449752816156) + | > loss_disc_real_1: 0.2249625027179718 (0.2238252289777945) + | > loss_disc_real_2: 0.23583072423934937 (0.22621931410689655) + | > loss_disc_real_3: 0.17155148088932037 (0.23544933164579482) + | > loss_disc_real_4: 0.20191162824630737 (0.23577321536284834) + | > loss_disc_real_5: 0.21397832036018372 (0.2359943611973245) + | > loss_0: 2.7470543384552 (2.673948842960286) + | > grad_norm_0: tensor(7.0885, device='cuda:0') (tensor(11.5699, device='cuda:0')) + | > loss_gen: 1.8313086032867432 (1.9741806747499577) + | > loss_kl: 2.5066425800323486 (2.3893503623888845) + | > loss_feat: 2.7330071926116943 (3.0730588764526936) + | > loss_mel: 24.72968101501465 (25.03519156484894) + | > loss_duration: 1.4853827953338623 (1.454499617752622) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 33.2860221862793 (33.92628094381537) + | > grad_norm_1: tensor(107.5899, device='cuda:0') (tensor(124.8291, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.9328 (0.8664188752818042) + | > loader_time: 0.0164 (0.013052104261624257) + + + --> TIME: 2024-06-25 04:03:26 -- STEP: 388/591 -- GLOBAL_STEP: 4525 + | > loss_disc: 2.7373247146606445 (2.67694568326793) + | > loss_disc_real_0: 0.15871527791023254 (0.21687057643130267) + | > loss_disc_real_1: 0.24500426650047302 (0.2237679348855289) + | > loss_disc_real_2: 0.20079372823238373 (0.22651494816713727) + | > loss_disc_real_3: 0.2103617936372757 (0.23569755862975858) + | > loss_disc_real_4: 0.23030243813991547 (0.23602586892462268) + | > loss_disc_real_5: 0.24794121086597443 (0.2362597445314078) + | > loss_0: 2.7373247146606445 (2.67694568326793) + | > grad_norm_0: tensor(11.1681, device='cuda:0') (tensor(11.6605, device='cuda:0')) + | > loss_gen: 1.9019498825073242 (1.971278485870853) + | > loss_kl: 2.2764155864715576 (2.3858289915261808) + | > loss_feat: 2.697646379470825 (3.058852381927449) + | > loss_mel: 24.658939361572266 (25.011872532441455) + | > loss_duration: 1.4451897144317627 (1.455549857358343) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 32.980140686035156 (33.88338208935927) + | > grad_norm_1: tensor(142.7146, device='cuda:0') (tensor(125.0669, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.957 (0.8728446573326268) + | > loader_time: 0.0156 (0.013306562433537752) + + + --> TIME: 2024-06-25 04:03:51 -- STEP: 413/591 -- GLOBAL_STEP: 4550 + | > loss_disc: 2.78806734085083 (2.6782234355843406) + | > loss_disc_real_0: 0.15197601914405823 (0.2168677797401211) + | > loss_disc_real_1: 0.22644354403018951 (0.22361022342203893) + | > loss_disc_real_2: 0.2219112068414688 (0.22659499614925707) + | > loss_disc_real_3: 0.2612578570842743 (0.23564884513977366) + | > loss_disc_real_4: 0.2311803102493286 (0.2359212933166841) + | > loss_disc_real_5: 0.23308506608009338 (0.23633888363838196) + | > loss_0: 2.78806734085083 (2.6782234355843406) + | > grad_norm_0: tensor(15.2911, device='cuda:0') (tensor(11.6390, device='cuda:0')) + | > loss_gen: 2.1174447536468506 (1.9689825617372267) + | > loss_kl: 2.277700662612915 (2.383649103289367) + | > loss_feat: 2.8060975074768066 (3.048775320768932) + | > loss_mel: 24.704322814941406 (24.98372292691802) + | > loss_duration: 1.4428709745407104 (1.4563604772235237) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 33.34843444824219 (33.84149023984306) + | > grad_norm_1: tensor(96.3874, device='cuda:0') (tensor(125.9300, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 0.9702 (0.8791714164881672) + | > loader_time: 0.0161 (0.01357162431712301) + + + --> TIME: 2024-06-25 04:04:16 -- STEP: 438/591 -- GLOBAL_STEP: 4575 + | > loss_disc: 2.713024854660034 (2.6818048964896724) + | > loss_disc_real_0: 0.17978286743164062 (0.21714196008838474) + | > loss_disc_real_1: 0.19318300485610962 (0.223771597254494) + | > loss_disc_real_2: 0.2285202294588089 (0.22709701723976222) + | > loss_disc_real_3: 0.26446598768234253 (0.2360926501827153) + | > loss_disc_real_4: 0.23093578219413757 (0.2360544798717107) + | > loss_disc_real_5: 0.24357396364212036 (0.23655344314498988) + | > loss_0: 2.713024854660034 (2.6818048964896724) + | > grad_norm_0: tensor(7.4595, device='cuda:0') (tensor(11.6740, device='cuda:0')) + | > loss_gen: 1.838315486907959 (1.9657903161223076) + | > loss_kl: 2.309583902359009 (2.381203565967682) + | > loss_feat: 2.829634428024292 (3.0355502372462984) + | > loss_mel: 25.09058380126953 (24.974714658031733) + | > loss_duration: 1.4979342222213745 (1.4572289958936442) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 33.56604766845703 (33.81448761404379) + | > grad_norm_1: tensor(151.4576, device='cuda:0') (tensor(128.1139, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 1.0166 (0.8860007399293386) + | > loader_time: 0.0193 (0.013830581756487286) + + + --> TIME: 2024-06-25 04:04:43 -- STEP: 463/591 -- GLOBAL_STEP: 4600 + | > loss_disc: 2.7595713138580322 (2.683808168118769) + | > loss_disc_real_0: 0.15060682594776154 (0.21709754453171407) + | > loss_disc_real_1: 0.2690034806728363 (0.22427874555726804) + | > loss_disc_real_2: 0.21491467952728271 (0.22742015580379166) + | > loss_disc_real_3: 0.194419726729393 (0.23593438209133066) + | > loss_disc_real_4: 0.2429567277431488 (0.23607192212639308) + | > loss_disc_real_5: 0.2875627279281616 (0.23657170662225965) + | > loss_0: 2.7595713138580322 (2.683808168118769) + | > grad_norm_0: tensor(13.3931, device='cuda:0') (tensor(11.6299, device='cuda:0')) + | > loss_gen: 1.8641325235366821 (1.9630169273710147) + | > loss_kl: 2.2977426052093506 (2.3782737002774397) + | > loss_feat: 2.7887518405914307 (3.021317128228831) + | > loss_mel: 24.271387100219727 (24.941872759462697) + | > loss_duration: 1.4672654867172241 (1.458157079781106) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 32.68927764892578 (33.7626374535118) + | > grad_norm_1: tensor(70.6307, device='cuda:0') (tensor(127.9437, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 1.016 (0.8932205600820916) + | > loader_time: 0.0201 (0.014125198833885771) + + + --> TIME: 2024-06-25 04:05:09 -- STEP: 488/591 -- GLOBAL_STEP: 4625 + | > loss_disc: 2.806412696838379 (2.6846638468445345) + | > loss_disc_real_0: 0.1918671727180481 (0.21713030339814113) + | > loss_disc_real_1: 0.28119367361068726 (0.22409902671810056) + | > loss_disc_real_2: 0.3224408030509949 (0.22765553247977477) + | > loss_disc_real_3: 0.29065871238708496 (0.23589593849954058) + | > loss_disc_real_4: 0.2595451772212982 (0.23603149534004633) + | > loss_disc_real_5: 0.2517118453979492 (0.23654280123529864) + | > loss_0: 2.806412696838379 (2.6846638468445345) + | > grad_norm_0: tensor(8.3283, device='cuda:0') (tensor(11.6117, device='cuda:0')) + | > loss_gen: 1.881147027015686 (1.9609992555418954) + | > loss_kl: 2.3288137912750244 (2.376447830043855) + | > loss_feat: 2.6438164710998535 (3.011806740135442) + | > loss_mel: 24.07903289794922 (24.921368716193033) + | > loss_duration: 1.5093421936035156 (1.4594729449905335) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 32.44215393066406 (33.730095370871155) + | > grad_norm_1: tensor(91.3412, device='cuda:0') (tensor(128.8175, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 1.0461 (0.9005037516844078) + | > loader_time: 0.0258 (0.014427413705919611) + + + --> TIME: 2024-06-25 04:05:37 -- STEP: 513/591 -- GLOBAL_STEP: 4650 + | > loss_disc: 2.7393786907196045 (2.6858215912973225) + | > loss_disc_real_0: 0.2181861400604248 (0.2171278340164681) + | > loss_disc_real_1: 0.17636670172214508 (0.22403710688415326) + | > loss_disc_real_2: 0.2529909014701843 (0.22782970671532912) + | > loss_disc_real_3: 0.31134918332099915 (0.236111661176003) + | > loss_disc_real_4: 0.29045096039772034 (0.23612290545281378) + | > loss_disc_real_5: 0.25052139163017273 (0.23664060917630297) + | > loss_0: 2.7393786907196045 (2.6858215912973225) + | > grad_norm_0: tensor(7.1425, device='cuda:0') (tensor(11.6203, device='cuda:0')) + | > loss_gen: 1.8461809158325195 (1.9589420654620344) + | > loss_kl: 2.297511339187622 (2.372745478594745) + | > loss_feat: 2.7076632976531982 (3.0027858797337332) + | > loss_mel: 24.292522430419922 (24.894692980522773) + | > loss_duration: 1.4818878173828125 (1.4609859452145375) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 32.62576675415039 (33.690152255647625) + | > grad_norm_1: tensor(158.5298, device='cuda:0') (tensor(130.0484, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 1.1006 (0.9099779254511783) + | > loader_time: 0.0231 (0.01469477807801602) + + + --> TIME: 2024-06-25 04:06:06 -- STEP: 538/591 -- GLOBAL_STEP: 4675 + | > loss_disc: 2.7553484439849854 (2.687577979715342) + | > loss_disc_real_0: 0.1478034257888794 (0.21732256867573166) + | > loss_disc_real_1: 0.18999351561069489 (0.22412086768442813) + | > loss_disc_real_2: 0.18649648129940033 (0.2279270754704688) + | > loss_disc_real_3: 0.22057496011257172 (0.23602946016757462) + | > loss_disc_real_4: 0.26461338996887207 (0.2361877576403015) + | > loss_disc_real_5: 0.23200112581253052 (0.23650411020645862) + | > loss_0: 2.7553484439849854 (2.687577979715342) + | > grad_norm_0: tensor(12.5661, device='cuda:0') (tensor(11.7130, device='cuda:0')) + | > loss_gen: 2.0287725925445557 (1.957704521466365) + | > loss_kl: 2.313776731491089 (2.3699461602810143) + | > loss_feat: 2.7533178329467773 (2.995758806905781) + | > loss_mel: 24.654193878173828 (24.86973107703114) + | > loss_duration: 1.5001354217529297 (1.4623003489022806) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 33.25019836425781 (33.655440837477244) + | > grad_norm_1: tensor(169.7096, device='cuda:0') (tensor(130.5716, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 1.1335 (0.9209338603852849) + | > loader_time: 0.02 (0.015024225951127403) + + + --> TIME: 2024-06-25 04:06:37 -- STEP: 563/591 -- GLOBAL_STEP: 4700 + | > loss_disc: 2.681366205215454 (2.6881052473722105) + | > loss_disc_real_0: 0.15410742163658142 (0.2173222295185601) + | > loss_disc_real_1: 0.23752444982528687 (0.22409974626177787) + | > loss_disc_real_2: 0.30436068773269653 (0.22818021500089453) + | > loss_disc_real_3: 0.2506941854953766 (0.23611945199712345) + | > loss_disc_real_4: 0.23785783350467682 (0.23618231666130451) + | > loss_disc_real_5: 0.246484637260437 (0.23654342131758668) + | > loss_0: 2.681366205215454 (2.6881052473722105) + | > grad_norm_0: tensor(11.1477, device='cuda:0') (tensor(11.6859, device='cuda:0')) + | > loss_gen: 1.887738823890686 (1.9557930008767976) + | > loss_kl: 2.33254337310791 (2.3672779385615836) + | > loss_feat: 2.784343957901001 (2.9881434394030104) + | > loss_mel: 23.574308395385742 (24.843059499140956) + | > loss_duration: 1.4662294387817383 (1.4635804730664355) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 32.045166015625 (33.61785427926916) + | > grad_norm_1: tensor(153.0089, device='cuda:0') (tensor(130.1820, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 1.3004 (0.9332087543887313) + | > loader_time: 0.0251 (0.015391154992220457) + + + --> TIME: 2024-06-25 04:07:09 -- STEP: 588/591 -- GLOBAL_STEP: 4725 + | > loss_disc: 2.698428153991699 (2.689175192596153) + | > loss_disc_real_0: 0.27194470167160034 (0.21735232915146424) + | > loss_disc_real_1: 0.2284146547317505 (0.22427074499681687) + | > loss_disc_real_2: 0.2204306572675705 (0.22820301143591906) + | > loss_disc_real_3: 0.23107506334781647 (0.23617283465201352) + | > loss_disc_real_4: 0.22868910431861877 (0.23616182241512804) + | > loss_disc_real_5: 0.24571193754673004 (0.2365514932542431) + | > loss_0: 2.698428153991699 (2.689175192596153) + | > grad_norm_0: tensor(14.0877, device='cuda:0') (tensor(11.6768, device='cuda:0')) + | > loss_gen: 1.866441011428833 (1.9535400449824172) + | > loss_kl: 2.303313970565796 (2.3653449537802724) + | > loss_feat: 2.5119383335113525 (2.979377764422875) + | > loss_mel: 23.216556549072266 (24.81920015568636) + | > loss_duration: 1.4934933185577393 (1.465219679738389) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 31.39174461364746 (33.58268250575686) + | > grad_norm_1: tensor(83.1146, device='cuda:0') (tensor(129.5888, device='cuda:0')) + | > current_lr_0: 0.00019982506561132978 + | > current_lr_1: 0.00019982506561132978 + | > step_time: 1.265 (0.9459440095895002) + | > loader_time: 0.0223 (0.01576525135105159) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036689169863437085 (-2.2183073327896544e-05) + | > avg_loss_disc: 2.7732084710547267 (+0.08971558733189333) + | > avg_loss_disc_real_0: 0.21320675345177345 (+0.05259144290330561) + | > avg_loss_disc_real_1: 0.23634177256137767 (+0.048209670059224385) + | > avg_loss_disc_real_2: 0.19093577595467262 (-0.0003281099365112783) + | > avg_loss_disc_real_3: 0.20893087095402657 (-0.013322093385331163) + | > avg_loss_disc_real_4: 0.21875936078264357 (+0.001008664039855306) + | > avg_loss_disc_real_5: 0.2109635903480205 (-0.016226343652035347) + | > avg_loss_0: 2.7732084710547267 (+0.08971558733189333) + | > avg_loss_gen: 1.7038623069195038 (+0.000986761235176381) + | > avg_loss_kl: 2.400454564297453 (+0.03965418643139795) + | > avg_loss_feat: 2.8309742339113924 (-0.14117339824108388) + | > avg_loss_mel: 24.934632199875846 (-0.1131313405138421) + | > avg_loss_duration: 1.4414110082261107 (-0.01529502361378765) + | > avg_loss_1: 33.31133469114911 (-0.228958413956029) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_4728.pth + + > EPOCH: 9/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 04:07:24)  + + --> TIME: 2024-06-25 04:07:44 -- STEP: 22/591 -- GLOBAL_STEP: 4750 + | > loss_disc: 2.7538704872131348 (2.72052124413577) + | > loss_disc_real_0: 0.3184862434864044 (0.21995874562046744) + | > loss_disc_real_1: 0.28616008162498474 (0.2268019447272474) + | > loss_disc_real_2: 0.2345820516347885 (0.2381199530579827) + | > loss_disc_real_3: 0.23791716992855072 (0.23873163624243302) + | > loss_disc_real_4: 0.24915094673633575 (0.23731733045794748) + | > loss_disc_real_5: 0.2835310697555542 (0.2414484443989667) + | > loss_0: 2.7538704872131348 (2.72052124413577) + | > grad_norm_0: tensor(16.6777, device='cuda:0') (tensor(10.7432, device='cuda:0')) + | > loss_gen: 1.8017057180404663 (1.9168844331394543) + | > loss_kl: 2.3758389949798584 (2.535931207916953) + | > loss_feat: 3.164346694946289 (3.0047739202325996) + | > loss_mel: 24.422483444213867 (24.644419410011984) + | > loss_duration: 1.4105548858642578 (1.4083254554054954) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 33.174930572509766 (33.510334708473906) + | > grad_norm_1: tensor(64.4370, device='cuda:0') (tensor(98.2813, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.7907 (0.7774996107274835) + | > loader_time: 0.0092 (0.008965828201987526) + + + --> TIME: 2024-06-25 04:08:05 -- STEP: 47/591 -- GLOBAL_STEP: 4775 + | > loss_disc: 2.77608060836792 (2.72799965675841) + | > loss_disc_real_0: 0.34885382652282715 (0.22114972453168097) + | > loss_disc_real_1: 0.24613407254219055 (0.22592029451055728) + | > loss_disc_real_2: 0.2815951406955719 (0.23752116300958268) + | > loss_disc_real_3: 0.26050519943237305 (0.23717907229636578) + | > loss_disc_real_4: 0.2703048586845398 (0.23794596436175894) + | > loss_disc_real_5: 0.22751154005527496 (0.23865133365418048) + | > loss_0: 2.77608060836792 (2.72799965675841) + | > grad_norm_0: tensor(21.1504, device='cuda:0') (tensor(12.1121, device='cuda:0')) + | > loss_gen: 1.8487237691879272 (1.9194210691654936) + | > loss_kl: 2.418569564819336 (2.5010349547609367) + | > loss_feat: 2.9610702991485596 (2.9635020773461522) + | > loss_mel: 24.86712646484375 (24.62001796478921) + | > loss_duration: 1.4357651472091675 (1.4234301841005366) + | > amp_scaler: 512.0 (969.531914893617) + | > loss_1: 33.531253814697266 (33.427406148707625) + | > grad_norm_1: tensor(165.5549, device='cuda:0') (tensor(122.0040, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.8033 (0.7903868137521947) + | > loader_time: 0.0101 (0.009626236367732922) + + + --> TIME: 2024-06-25 04:08:25 -- STEP: 72/591 -- GLOBAL_STEP: 4800 + | > loss_disc: 2.7622132301330566 (2.7294088767634497) + | > loss_disc_real_0: 0.1674553006887436 (0.21861428850226933) + | > loss_disc_real_1: 0.2095831334590912 (0.2282448559999466) + | > loss_disc_real_2: 0.22299860417842865 (0.23594280725551975) + | > loss_disc_real_3: 0.2531457245349884 (0.2368474060462581) + | > loss_disc_real_4: 0.25297388434410095 (0.23776181745860311) + | > loss_disc_real_5: 0.23881249129772186 (0.2371612967302402) + | > loss_0: 2.7622132301330566 (2.7294088767634497) + | > grad_norm_0: tensor(11.3957, device='cuda:0') (tensor(11.1436, device='cuda:0')) + | > loss_gen: 1.9534610509872437 (1.9089682151873906) + | > loss_kl: 2.5284197330474854 (2.4666441712114544) + | > loss_feat: 3.036555767059326 (2.941607634226481) + | > loss_mel: 24.932140350341797 (24.60073669751485) + | > loss_duration: 1.437263011932373 (1.4314463933308919) + | > amp_scaler: 512.0 (810.6666666666666) + | > loss_1: 33.887840270996094 (33.349402931001464) + | > grad_norm_1: tensor(155.7939, device='cuda:0') (tensor(129.7399, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.8282 (0.7994819978872935) + | > loader_time: 0.0114 (0.009995672437879775) + + + --> TIME: 2024-06-25 04:08:46 -- STEP: 97/591 -- GLOBAL_STEP: 4825 + | > loss_disc: 2.7447497844696045 (2.730666357217375) + | > loss_disc_real_0: 0.16944339871406555 (0.21949922147485398) + | > loss_disc_real_1: 0.25968772172927856 (0.22738594031825507) + | > loss_disc_real_2: 0.2728370726108551 (0.2354824728879732) + | > loss_disc_real_3: 0.25227680802345276 (0.2369144489777457) + | > loss_disc_real_4: 0.24525727331638336 (0.23711671657168987) + | > loss_disc_real_5: 0.25184890627861023 (0.23688749462058864) + | > loss_0: 2.7447497844696045 (2.730666357217375) + | > grad_norm_0: tensor(8.7185, device='cuda:0') (tensor(11.1804, device='cuda:0')) + | > loss_gen: 2.132906198501587 (1.9071779582918305) + | > loss_kl: 2.4507367610931396 (2.45062287566588) + | > loss_feat: 2.837066650390625 (2.933767163876405) + | > loss_mel: 24.535442352294922 (24.580245853699356) + | > loss_duration: 1.4389631748199463 (1.440405962393456) + | > amp_scaler: 512.0 (733.6907216494845) + | > loss_1: 33.39511489868164 (33.31221971806792) + | > grad_norm_1: tensor(75.0399, device='cuda:0') (tensor(131.9478, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.8291 (0.8070424659964964) + | > loader_time: 0.0105 (0.010436547171209273) + + + --> TIME: 2024-06-25 04:09:08 -- STEP: 122/591 -- GLOBAL_STEP: 4850 + | > loss_disc: 2.683218479156494 (2.7319827861473205) + | > loss_disc_real_0: 0.16103452444076538 (0.22028903883011614) + | > loss_disc_real_1: 0.18466082215309143 (0.22737477390003985) + | > loss_disc_real_2: 0.2013830691576004 (0.23491366659520102) + | > loss_disc_real_3: 0.19589164853096008 (0.23710839743496942) + | > loss_disc_real_4: 0.22641263902187347 (0.2367403684336631) + | > loss_disc_real_5: 0.20754295587539673 (0.23685066817236727) + | > loss_0: 2.683218479156494 (2.7319827861473205) + | > grad_norm_0: tensor(9.7361, device='cuda:0') (tensor(11.1559, device='cuda:0')) + | > loss_gen: 1.9639604091644287 (1.903254406374009) + | > loss_kl: 2.4646174907684326 (2.431651263940529) + | > loss_feat: 3.0938963890075684 (2.9275337340401815) + | > loss_mel: 24.793886184692383 (24.62023366083864) + | > loss_duration: 1.4562962055206299 (1.4426221271030237) + | > amp_scaler: 512.0 (688.2622950819672) + | > loss_1: 33.77265548706055 (33.32529499491709) + | > grad_norm_1: tensor(104.0677, device='cuda:0') (tensor(131.4291, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.8563 (0.8141944408416748) + | > loader_time: 0.0129 (0.010707265040913565) + + + --> TIME: 2024-06-25 04:09:30 -- STEP: 147/591 -- GLOBAL_STEP: 4875 + | > loss_disc: 2.6353816986083984 (2.728683637112986) + | > loss_disc_real_0: 0.2133924663066864 (0.2199007082147664) + | > loss_disc_real_1: 0.19344894587993622 (0.2265373936721257) + | > loss_disc_real_2: 0.27594470977783203 (0.23499045394310336) + | > loss_disc_real_3: 0.18282251060009003 (0.23718544192054644) + | > loss_disc_real_4: 0.18691127002239227 (0.23654738950486087) + | > loss_disc_real_5: 0.1968662440776825 (0.23689830688392224) + | > loss_0: 2.6353816986083984 (2.728683637112986) + | > grad_norm_0: tensor(4.8330, device='cuda:0') (tensor(10.9424, device='cuda:0')) + | > loss_gen: 1.9756507873535156 (1.9047384237756535) + | > loss_kl: 2.3101043701171875 (2.4273228888608966) + | > loss_feat: 3.1567821502685547 (2.935922609705503) + | > loss_mel: 24.13222885131836 (24.598698246235745) + | > loss_duration: 1.4103902578353882 (1.4423839186324554) + | > amp_scaler: 512.0 (658.2857142857143) + | > loss_1: 32.98515701293945 (33.309065864199695) + | > grad_norm_1: tensor(84.7038, device='cuda:0') (tensor(132.5676, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.846 (0.8218499267993328) + | > loader_time: 0.0119 (0.010883329676933026) + + + --> TIME: 2024-06-25 04:09:52 -- STEP: 172/591 -- GLOBAL_STEP: 4900 + | > loss_disc: 2.702254295349121 (2.7270113451536293) + | > loss_disc_real_0: 0.1744585633277893 (0.22002560116870468) + | > loss_disc_real_1: 0.2487156093120575 (0.2257095714814441) + | > loss_disc_real_2: 0.2119709551334381 (0.23466184345442195) + | > loss_disc_real_3: 0.20865796506404877 (0.2372857968128005) + | > loss_disc_real_4: 0.24878577888011932 (0.2366788639405439) + | > loss_disc_real_5: 0.25588667392730713 (0.23676567282094513) + | > loss_0: 2.702254295349121 (2.7270113451536293) + | > grad_norm_0: tensor(7.7031, device='cuda:0') (tensor(10.9069, device='cuda:0')) + | > loss_gen: 1.947424054145813 (1.9060972818108493) + | > loss_kl: 2.349111557006836 (2.420096351656803) + | > loss_feat: 2.6684625148773193 (2.9367963103360912) + | > loss_mel: 24.437602996826172 (24.55923306664754) + | > loss_duration: 1.4198814630508423 (1.4403800014839612) + | > amp_scaler: 512.0 (637.0232558139536) + | > loss_1: 32.82248306274414 (33.262602916983695) + | > grad_norm_1: tensor(185.4333, device='cuda:0') (tensor(131.6618, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.8648 (0.8273302247357923) + | > loader_time: 0.0116 (0.010999676793120625) + + + --> TIME: 2024-06-25 04:10:14 -- STEP: 197/591 -- GLOBAL_STEP: 4925 + | > loss_disc: 2.6770896911621094 (2.725109536030569) + | > loss_disc_real_0: 0.13565948605537415 (0.22018132513852295) + | > loss_disc_real_1: 0.20799319446086884 (0.22551829333837867) + | > loss_disc_real_2: 0.2455788105726242 (0.23440041850666105) + | > loss_disc_real_3: 0.20944932103157043 (0.23701175948992598) + | > loss_disc_real_4: 0.210251584649086 (0.23600972311448326) + | > loss_disc_real_5: 0.24004250764846802 (0.23664756580657764) + | > loss_0: 2.6770896911621094 (2.725109536030569) + | > grad_norm_0: tensor(14.8488, device='cuda:0') (tensor(11.1487, device='cuda:0')) + | > loss_gen: 1.8268924951553345 (1.9070538827005377) + | > loss_kl: 2.3289692401885986 (2.4180228334998115) + | > loss_feat: 3.130307674407959 (2.9419265543748883) + | > loss_mel: 24.49759864807129 (24.55607603044073) + | > loss_duration: 1.4481427669525146 (1.4397353321162578) + | > amp_scaler: 512.0 (621.1573604060917) + | > loss_1: 33.231910705566406 (33.262814536312526) + | > grad_norm_1: tensor(147.3856, device='cuda:0') (tensor(133.5222, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.8738 (0.8325400667142142) + | > loader_time: 0.0117 (0.011168673558888698) + + + --> TIME: 2024-06-25 04:10:36 -- STEP: 222/591 -- GLOBAL_STEP: 4950 + | > loss_disc: 2.7861735820770264 (2.726418860323793) + | > loss_disc_real_0: 0.33315309882164 (0.2207717218914548) + | > loss_disc_real_1: 0.25262489914894104 (0.22605766973516964) + | > loss_disc_real_2: 0.24498632550239563 (0.23412165193407386) + | > loss_disc_real_3: 0.381515234708786 (0.23754449953904022) + | > loss_disc_real_4: 0.23827947676181793 (0.2362592122307769) + | > loss_disc_real_5: 0.2555537521839142 (0.23654128100957955) + | > loss_0: 2.7861735820770264 (2.726418860323793) + | > grad_norm_0: tensor(18.3336, device='cuda:0') (tensor(11.0440, device='cuda:0')) + | > loss_gen: 1.8152756690979004 (1.9066419236294858) + | > loss_kl: 2.500835418701172 (2.4184052471642024) + | > loss_feat: 2.673828125 (2.9330495477796674) + | > loss_mel: 23.596878051757812 (24.54599845301997) + | > loss_duration: 1.4439672231674194 (1.4388070203162528) + | > amp_scaler: 512.0 (608.8648648648651) + | > loss_1: 32.030784606933594 (33.242902042629495) + | > grad_norm_1: tensor(168.3016, device='cuda:0') (tensor(133.4705, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.8734 (0.8375024945886286) + | > loader_time: 0.0128 (0.011427912626180559) + + + --> TIME: 2024-06-25 04:10:59 -- STEP: 247/591 -- GLOBAL_STEP: 4975 + | > loss_disc: 2.64398193359375 (2.7254699454133795) + | > loss_disc_real_0: 0.2421749085187912 (0.22042031230231535) + | > loss_disc_real_1: 0.23738929629325867 (0.22589232184385) + | > loss_disc_real_2: 0.2730327546596527 (0.2343020345638638) + | > loss_disc_real_3: 0.26618310809135437 (0.23727432701751772) + | > loss_disc_real_4: 0.24146075546741486 (0.23619485432319795) + | > loss_disc_real_5: 0.25223952531814575 (0.23650547222569887) + | > loss_0: 2.64398193359375 (2.7254699454133795) + | > grad_norm_0: tensor(6.6309, device='cuda:0') (tensor(11.0643, device='cuda:0')) + | > loss_gen: 1.934385061264038 (1.90798352942293) + | > loss_kl: 2.370836019515991 (2.415798746139897) + | > loss_feat: 3.093312978744507 (2.934392739886697) + | > loss_mel: 24.739849090576172 (24.546383336488045) + | > loss_duration: 1.4564015865325928 (1.4391733760293197) + | > amp_scaler: 512.0 (599.0607287449395) + | > loss_1: 33.594783782958984 (33.24373156435578) + | > grad_norm_1: tensor(198.1272, device='cuda:0') (tensor(137.2959, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.9001 (0.8424943526264145) + | > loader_time: 0.0151 (0.011746378562711025) + + + --> TIME: 2024-06-25 04:11:22 -- STEP: 272/591 -- GLOBAL_STEP: 5000 + | > loss_disc: 2.763791084289551 (2.7292616078082244) + | > loss_disc_real_0: 0.3211677670478821 (0.22089787506881886) + | > loss_disc_real_1: 0.16657890379428864 (0.22714219116331902) + | > loss_disc_real_2: 0.1938859224319458 (0.23446334865601623) + | > loss_disc_real_3: 0.22023579478263855 (0.23710654100731893) + | > loss_disc_real_4: 0.16984790563583374 (0.23622710876824224) + | > loss_disc_real_5: 0.17973263561725616 (0.23651247789316318) + | > loss_0: 2.763791084289551 (2.7292616078082244) + | > grad_norm_0: tensor(13.9512, device='cuda:0') (tensor(11.1362, device='cuda:0')) + | > loss_gen: 1.694281816482544 (1.9063156790593092) + | > loss_kl: 2.2242636680603027 (2.413242805530043) + | > loss_feat: 2.7867584228515625 (2.9264096679056393) + | > loss_mel: 24.203725814819336 (24.541004475425265) + | > loss_duration: 1.4643545150756836 (1.4389803006368524) + | > amp_scaler: 512.0 (591.0588235294119) + | > loss_1: 32.37338638305664 (33.2259527725332) + | > grad_norm_1: tensor(126.1081, device='cuda:0') (tensor(136.0231, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.915 (0.8476723879575729) + | > loader_time: 0.0159 (0.012025468489703007) + + + > CHECKPOINT : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/checkpoint_5000.pth + + --> TIME: 2024-06-25 04:11:48 -- STEP: 297/591 -- GLOBAL_STEP: 5025 + | > loss_disc: 2.6991264820098877 (2.7297109682551945) + | > loss_disc_real_0: 0.18472149968147278 (0.22084466164762334) + | > loss_disc_real_1: 0.17543314397335052 (0.22697189505453463) + | > loss_disc_real_2: 0.23001085221767426 (0.2344549400637848) + | > loss_disc_real_3: 0.18810003995895386 (0.23692191760949413) + | > loss_disc_real_4: 0.20410273969173431 (0.23624655810068754) + | > loss_disc_real_5: 0.20753230154514313 (0.2366114450424207) + | > loss_0: 2.6991264820098877 (2.7297109682551945) + | > grad_norm_0: tensor(7.4917, device='cuda:0') (tensor(10.8925, device='cuda:0')) + | > loss_gen: 2.0388684272766113 (1.904574919228602) + | > loss_kl: 2.453305721282959 (2.4131629611506606) + | > loss_feat: 3.1082406044006348 (2.91788744203972) + | > loss_mel: 24.67453384399414 (24.511039643978023) + | > loss_duration: 1.425191879272461 (1.4393181523891414) + | > amp_scaler: 512.0 (584.4040404040405) + | > loss_1: 33.70014190673828 (33.18598300278789) + | > grad_norm_1: tensor(100.4568, device='cuda:0') (tensor(135.1248, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.903 (0.8529143799033629) + | > loader_time: 0.014 (0.01228235947965371) + + + --> TIME: 2024-06-25 04:12:12 -- STEP: 322/591 -- GLOBAL_STEP: 5050 + | > loss_disc: 2.705080270767212 (2.731085371526873) + | > loss_disc_real_0: 0.2871014475822449 (0.22138438040610434) + | > loss_disc_real_1: 0.19571666419506073 (0.22689284106589253) + | > loss_disc_real_2: 0.2310728281736374 (0.23439503183461124) + | > loss_disc_real_3: 0.2677072286605835 (0.23720285680538378) + | > loss_disc_real_4: 0.23359374701976776 (0.23633892463971368) + | > loss_disc_real_5: 0.26808035373687744 (0.2368419357338307) + | > loss_0: 2.705080270767212 (2.731085371526873) + | > grad_norm_0: tensor(13.5487, device='cuda:0') (tensor(10.8631, device='cuda:0')) + | > loss_gen: 2.0125839710235596 (1.9037401953839368) + | > loss_kl: 2.34523868560791 (2.412202688477795) + | > loss_feat: 2.933403730392456 (2.9137301615306304) + | > loss_mel: 24.766260147094727 (24.497214394326537) + | > loss_duration: 1.4002978801727295 (1.439868477560719) + | > amp_scaler: 512.0 (578.7826086956525) + | > loss_1: 33.457786560058594 (33.166755824355604) + | > grad_norm_1: tensor(47.0404, device='cuda:0') (tensor(133.3351, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.9725 (0.8582139444647369) + | > loader_time: 0.0139 (0.012505000422459946) + + + --> TIME: 2024-06-25 04:12:36 -- STEP: 347/591 -- GLOBAL_STEP: 5075 + | > loss_disc: 2.7704274654388428 (2.7317781118563365) + | > loss_disc_real_0: 0.12862688302993774 (0.22123146136573146) + | > loss_disc_real_1: 0.2590557336807251 (0.22714125354283138) + | > loss_disc_real_2: 0.2573583126068115 (0.23482311188487567) + | > loss_disc_real_3: 0.27247685194015503 (0.23726701813747286) + | > loss_disc_real_4: 0.2644200026988983 (0.23632059536165737) + | > loss_disc_real_5: 0.24081923067569733 (0.23689423471431567) + | > loss_0: 2.7704274654388428 (2.7317781118563365) + | > grad_norm_0: tensor(17.0632, device='cuda:0') (tensor(10.9730, device='cuda:0')) + | > loss_gen: 1.7689104080200195 (1.903037594786982) + | > loss_kl: 2.447211503982544 (2.408804966324689) + | > loss_feat: 2.720069169998169 (2.907685666675182) + | > loss_mel: 24.1168212890625 (24.481693537159682) + | > loss_duration: 1.4601635932922363 (1.4411777233871332) + | > amp_scaler: 512.0 (573.971181556196) + | > loss_1: 32.51317596435547 (33.14239942787017) + | > grad_norm_1: tensor(77.2618, device='cuda:0') (tensor(132.9525, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.9279 (0.8642898602169597) + | > loader_time: 0.0149 (0.012741648154574781) + + + --> TIME: 2024-06-25 04:13:00 -- STEP: 372/591 -- GLOBAL_STEP: 5100 + | > loss_disc: 2.7263264656066895 (2.7310556814234763) + | > loss_disc_real_0: 0.10728747397661209 (0.221193544005835) + | > loss_disc_real_1: 0.22631597518920898 (0.22688229073600102) + | > loss_disc_real_2: 0.22201547026634216 (0.23462070785062286) + | > loss_disc_real_3: 0.18279770016670227 (0.2371093499003559) + | > loss_disc_real_4: 0.26186975836753845 (0.23640576277368813) + | > loss_disc_real_5: 0.24520821869373322 (0.23699502118172183) + | > loss_0: 2.7263264656066895 (2.7310556814234763) + | > grad_norm_0: tensor(22.3775, device='cuda:0') (tensor(10.8900, device='cuda:0')) + | > loss_gen: 1.9253082275390625 (1.9021782548196855) + | > loss_kl: 2.3710076808929443 (2.405744879476488) + | > loss_feat: 2.880948305130005 (2.9028725162629154) + | > loss_mel: 24.67911720275879 (24.45789368947347) + | > loss_duration: 1.4579850435256958 (1.4426765025302934) + | > amp_scaler: 512.0 (569.8064516129032) + | > loss_1: 33.31436538696289 (33.11136575411725) + | > grad_norm_1: tensor(142.9128, device='cuda:0') (tensor(133.6133, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.9456 (0.8698984467855064) + | > loader_time: 0.0175 (0.01301196121400402) + + + --> TIME: 2024-06-25 04:13:25 -- STEP: 397/591 -- GLOBAL_STEP: 5125 + | > loss_disc: 2.6746950149536133 (2.7306647222648657) + | > loss_disc_real_0: 0.11951868236064911 (0.22139053718569304) + | > loss_disc_real_1: 0.2351251244544983 (0.22673679017450107) + | > loss_disc_real_2: 0.19460682570934296 (0.23462922360644833) + | > loss_disc_real_3: 0.1956671178340912 (0.23705575545728957) + | > loss_disc_real_4: 0.20491093397140503 (0.2361222451519906) + | > loss_disc_real_5: 0.22979439795017242 (0.23673734523036918) + | > loss_0: 2.6746950149536133 (2.7306647222648657) + | > grad_norm_0: tensor(19.1618, device='cuda:0') (tensor(10.9651, device='cuda:0')) + | > loss_gen: 1.9769954681396484 (1.901734165340587) + | > loss_kl: 2.303887128829956 (2.403125685468431) + | > loss_feat: 3.09430193901062 (2.9003836772303733) + | > loss_mel: 24.74239730834961 (24.447001320288823) + | > loss_duration: 1.4629770517349243 (1.4435018859522175) + | > amp_scaler: 512.0 (566.1662468513853) + | > loss_1: 33.580562591552734 (33.09574665290582) + | > grad_norm_1: tensor(150.3755, device='cuda:0') (tensor(134.1164, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.9882 (0.8765074347969266) + | > loader_time: 0.0179 (0.013290299576535929) + + + --> TIME: 2024-06-25 04:13:50 -- STEP: 422/591 -- GLOBAL_STEP: 5150 + | > loss_disc: 2.7287845611572266 (2.729579560564594) + | > loss_disc_real_0: 0.23205508291721344 (0.2213615393398498) + | > loss_disc_real_1: 0.22293666005134583 (0.22648254390011466) + | > loss_disc_real_2: 0.26182642579078674 (0.2344372586757651) + | > loss_disc_real_3: 0.18264272809028625 (0.23701416167885206) + | > loss_disc_real_4: 0.210447296500206 (0.2361013278788865) + | > loss_disc_real_5: 0.20320837199687958 (0.23684023034657348) + | > loss_0: 2.7287845611572266 (2.729579560564594) + | > grad_norm_0: tensor(7.2706, device='cuda:0') (tensor(10.8688, device='cuda:0')) + | > loss_gen: 1.8095084428787231 (1.9008811977802296) + | > loss_kl: 2.369173765182495 (2.3993261640105796) + | > loss_feat: 2.6126134395599365 (2.8969466064778544) + | > loss_mel: 24.581077575683594 (24.42622222267621) + | > loss_duration: 1.44370698928833 (1.444056414032435) + | > amp_scaler: 512.0 (562.9573459715637) + | > loss_1: 32.81608200073242 (33.06743257079642) + | > grad_norm_1: tensor(140.0005, device='cuda:0') (tensor(133.2245, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 0.9762 (0.8832661032111724) + | > loader_time: 0.0188 (0.013555647637606793) + + + --> TIME: 2024-06-25 04:14:16 -- STEP: 447/591 -- GLOBAL_STEP: 5175 + | > loss_disc: 2.820469856262207 (2.730376872973689) + | > loss_disc_real_0: 0.2693330943584442 (0.22139490360781658) + | > loss_disc_real_1: 0.24702242016792297 (0.22712236129464186) + | > loss_disc_real_2: 0.26003021001815796 (0.23451050062574263) + | > loss_disc_real_3: 0.19537253677845 (0.23691393531675572) + | > loss_disc_real_4: 0.2525915205478668 (0.23629920034600585) + | > loss_disc_real_5: 0.23434126377105713 (0.2368608370456653) + | > loss_0: 2.820469856262207 (2.730376872973689) + | > grad_norm_0: tensor(9.3300, device='cuda:0') (tensor(11.0347, device='cuda:0')) + | > loss_gen: 1.892519474029541 (1.9029285142352383) + | > loss_kl: 2.415048360824585 (2.3974661149701313) + | > loss_feat: 2.754462242126465 (2.8950195072481293) + | > loss_mel: 24.402294158935547 (24.40792579992239) + | > loss_duration: 1.4707610607147217 (1.445032003208562) + | > amp_scaler: 512.0 (560.107382550335) + | > loss_1: 32.93508529663086 (33.0483719102488) + | > grad_norm_1: tensor(168.6026, device='cuda:0') (tensor(133.2994, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 1.0493 (0.8902609140280909) + | > loader_time: 0.0204 (0.013846394199652955) + + + --> TIME: 2024-06-25 04:14:42 -- STEP: 472/591 -- GLOBAL_STEP: 5200 + | > loss_disc: 2.7185544967651367 (2.731003733004555) + | > loss_disc_real_0: 0.28502780199050903 (0.22161393377424807) + | > loss_disc_real_1: 0.19846844673156738 (0.22715861883835267) + | > loss_disc_real_2: 0.2138793170452118 (0.2344465622899391) + | > loss_disc_real_3: 0.22903908789157867 (0.2370522419716847) + | > loss_disc_real_4: 0.23843911290168762 (0.23628294461611973) + | > loss_disc_real_5: 0.24740412831306458 (0.23692532003683558) + | > loss_0: 2.7185544967651367 (2.731003733004555) + | > grad_norm_0: tensor(11.2143, device='cuda:0') (tensor(10.9935, device='cuda:0')) + | > loss_gen: 1.6899200677871704 (1.9014779033802323) + | > loss_kl: 2.3472039699554443 (2.395051079786433) + | > loss_feat: 2.7529256343841553 (2.8874375996953345) + | > loss_mel: 23.2833194732666 (24.373639369415024) + | > loss_duration: 1.4968774318695068 (1.4462090068954536) + | > amp_scaler: 512.0 (557.5593220338981) + | > loss_1: 31.570247650146484 (33.00381492760221) + | > grad_norm_1: tensor(193.9927, device='cuda:0') (tensor(133.3481, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 1.0083 (0.896812724359965) + | > loader_time: 0.0217 (0.01414188742637634) + + + --> TIME: 2024-06-25 04:15:09 -- STEP: 497/591 -- GLOBAL_STEP: 5225 + | > loss_disc: 2.7024567127227783 (2.7302393049781246) + | > loss_disc_real_0: 0.24772542715072632 (0.22147934522547238) + | > loss_disc_real_1: 0.2731287479400635 (0.227167674503336) + | > loss_disc_real_2: 0.17597883939743042 (0.2342747117671928) + | > loss_disc_real_3: 0.3028433322906494 (0.2369931991731137) + | > loss_disc_real_4: 0.20438583195209503 (0.23620896320227885) + | > loss_disc_real_5: 0.23244142532348633 (0.23692067660076518) + | > loss_0: 2.7024567127227783 (2.7302393049781246) + | > grad_norm_0: tensor(9.7793, device='cuda:0') (tensor(10.9906, device='cuda:0')) + | > loss_gen: 1.7777150869369507 (1.9018592095710862) + | > loss_kl: 2.3580775260925293 (2.392495340984355) + | > loss_feat: 2.913118839263916 (2.8867511418263914) + | > loss_mel: 23.790027618408203 (24.34904354486907) + | > loss_duration: 1.4725627899169922 (1.4474501806485585) + | > amp_scaler: 512.0 (555.2676056338025) + | > loss_1: 32.311500549316406 (32.97759938000195) + | > grad_norm_1: tensor(100.3234, device='cuda:0') (tensor(133.5653, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 1.1952 (0.9049431145550979) + | > loader_time: 0.0173 (0.014424612824345975) + + + --> TIME: 2024-06-25 04:15:38 -- STEP: 522/591 -- GLOBAL_STEP: 5250 + | > loss_disc: 2.7363991737365723 (2.7296710667482293) + | > loss_disc_real_0: 0.3262154161930084 (0.2213055513713552) + | > loss_disc_real_1: 0.20737506449222565 (0.22700179399658438) + | > loss_disc_real_2: 0.23709145188331604 (0.23430420072941943) + | > loss_disc_real_3: 0.2557559013366699 (0.23691255370890044) + | > loss_disc_real_4: 0.23688089847564697 (0.23626171366465046) + | > loss_disc_real_5: 0.21976177394390106 (0.23694289886745915) + | > loss_0: 2.7363991737365723 (2.7296710667482293) + | > grad_norm_0: tensor(18.4474, device='cuda:0') (tensor(11.0120, device='cuda:0')) + | > loss_gen: 1.8886797428131104 (1.9029060188837892) + | > loss_kl: 2.449925661087036 (2.3929693858742285) + | > loss_feat: 2.780824899673462 (2.888231136332983) + | > loss_mel: 23.356054306030273 (24.334246434471158) + | > loss_duration: 1.4733128547668457 (1.4488126870772853) + | > amp_scaler: 512.0 (553.1954022988504) + | > loss_1: 31.94879722595215 (32.967165632722946) + | > grad_norm_1: tensor(71.8249, device='cuda:0') (tensor(133.0452, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 1.0964 (0.9150300980527739) + | > loader_time: 0.0196 (0.014734533554749468) + + + --> TIME: 2024-06-25 04:16:08 -- STEP: 547/591 -- GLOBAL_STEP: 5275 + | > loss_disc: 2.809661626815796 (2.729119844802774) + | > loss_disc_real_0: 0.4516340494155884 (0.22131399927453108) + | > loss_disc_real_1: 0.2245025485754013 (0.226842021457238) + | > loss_disc_real_2: 0.2220710813999176 (0.23423682907817564) + | > loss_disc_real_3: 0.2635830342769623 (0.236905594194611) + | > loss_disc_real_4: 0.2389969676733017 (0.23617831417367924) + | > loss_disc_real_5: 0.2740127742290497 (0.23705550663214062) + | > loss_0: 2.809661626815796 (2.729119844802774) + | > grad_norm_0: tensor(37.7790, device='cuda:0') (tensor(10.9702, device='cuda:0')) + | > loss_gen: 1.698141098022461 (1.9021506368351158) + | > loss_kl: 2.387930393218994 (2.391997687341529) + | > loss_feat: 2.6614749431610107 (2.8842772822929255) + | > loss_mel: 22.867198944091797 (24.30368081394454) + | > loss_duration: 1.5103950500488281 (1.4502865167596886) + | > amp_scaler: 512.0 (551.3126142595976) + | > loss_1: 31.125139236450195 (32.93239290840666) + | > grad_norm_1: tensor(99.7328, device='cuda:0') (tensor(132.4735, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 1.1748 (0.9279402971703566) + | > loader_time: 0.0208 (0.01507098365309469) + + + --> TIME: 2024-06-25 04:16:41 -- STEP: 572/591 -- GLOBAL_STEP: 5300 + | > loss_disc: 2.7186734676361084 (2.729651271463275) + | > loss_disc_real_0: 0.17377948760986328 (0.22109712952083643) + | > loss_disc_real_1: 0.23194821178913116 (0.22686609349675946) + | > loss_disc_real_2: 0.2936936020851135 (0.23475496919005062) + | > loss_disc_real_3: 0.21890626847743988 (0.23676981524258225) + | > loss_disc_real_4: 0.22419507801532745 (0.23617839862729287) + | > loss_disc_real_5: 0.24309282004833221 (0.23703174853137324) + | > loss_0: 2.7186734676361084 (2.729651271463275) + | > grad_norm_0: tensor(8.1734, device='cuda:0') (tensor(11.0752, device='cuda:0')) + | > loss_gen: 2.0319340229034424 (1.9024794908253464) + | > loss_kl: 2.306633234024048 (2.3895996371349284) + | > loss_feat: 2.6595280170440674 (2.8788629987023087) + | > loss_mel: 23.880830764770508 (24.27051133042449) + | > loss_duration: 1.4839953184127808 (1.4517042199214862) + | > amp_scaler: 512.0 (549.5944055944052) + | > loss_1: 32.36292266845703 (32.89315765554256) + | > grad_norm_1: tensor(239.0855, device='cuda:0') (tensor(132.4564, device='cuda:0')) + | > current_lr_0: 0.00019980008747812837 + | > current_lr_1: 0.00019980008747812837 + | > step_time: 1.2448 (0.9427706336641645) + | > loader_time: 0.0269 (0.015461900851109642) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036763739078602893 (+7.456921516580794e-06) + | > avg_loss_disc: 2.7237548117942 (-0.049453659260526805) + | > avg_loss_disc_real_0: 0.25060355726708755 (+0.0373968038153141) + | > avg_loss_disc_real_1: 0.2617985029169853 (+0.025456730355607637) + | > avg_loss_disc_real_2: 0.2585064737086601 (+0.06757069775398747) + | > avg_loss_disc_real_3: 0.29891393793390153 (+0.08998306697987496) + | > avg_loss_disc_real_4: 0.2178030622766373 (-0.0009562985060062867) + | > avg_loss_disc_real_5: 0.23343509245426097 (+0.022471502106240465) + | > avg_loss_0: 2.7237548117942 (-0.049453659260526805) + | > avg_loss_gen: 2.0333885588544494 (+0.32952625193494556) + | > avg_loss_kl: 2.4401543698412302 (+0.03969980554377717) + | > avg_loss_feat: 2.864799111447436 (+0.03382487753604346) + | > avg_loss_mel: 23.85811919354378 (-1.0765130063320676) + | > avg_loss_duration: 1.4248839692866548 (-0.01652703893945584) + | > avg_loss_1: 32.62134519536445 (-0.6899894957846655) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_5319.pth + + > EPOCH: 10/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 04:17:17)  + + --> TIME: 2024-06-25 04:17:24 -- STEP: 6/591 -- GLOBAL_STEP: 5325 + | > loss_disc: 2.7106003761291504 (2.723697026570638) + | > loss_disc_real_0: 0.22457605600357056 (0.2123725712299347) + | > loss_disc_real_1: 0.24430401623249054 (0.22277181843916574) + | > loss_disc_real_2: 0.2492315024137497 (0.22800963620344797) + | > loss_disc_real_3: 0.22785016894340515 (0.24106906106074652) + | > loss_disc_real_4: 0.2065832018852234 (0.24907360970973969) + | > loss_disc_real_5: 0.2479589879512787 (0.2457755208015442) + | > loss_0: 2.7106003761291504 (2.723697026570638) + | > grad_norm_0: tensor(5.3069, device='cuda:0') (tensor(8.1612, device='cuda:0')) + | > loss_gen: 1.9427335262298584 (1.895359496275584) + | > loss_kl: 2.7917137145996094 (2.6481842597325644) + | > loss_feat: 3.0190560817718506 (3.00196107228597) + | > loss_mel: 24.415283203125 (24.156760851542156) + | > loss_duration: 1.4249407052993774 (1.4081785480181377) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.593727111816406 (33.110443115234375) + | > grad_norm_1: tensor(168.9287, device='cuda:0') (tensor(140.5045, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.7658 (0.7720111211140951) + | > loader_time: 0.0074 (0.0081025759379069) + + + --> TIME: 2024-06-25 04:17:44 -- STEP: 31/591 -- GLOBAL_STEP: 5350 + | > loss_disc: 2.9333057403564453 (2.7466598941433813) + | > loss_disc_real_0: 0.20606958866119385 (0.2188533092698743) + | > loss_disc_real_1: 0.42688682675361633 (0.23732126335943898) + | > loss_disc_real_2: 0.2807711064815521 (0.23666327038118917) + | > loss_disc_real_3: 0.2778591215610504 (0.23936187019271235) + | > loss_disc_real_4: 0.23355455696582794 (0.23954994399701396) + | > loss_disc_real_5: 0.23511120676994324 (0.23955263678104646) + | > loss_0: 2.9333057403564453 (2.7466598941433813) + | > grad_norm_0: tensor(9.2437, device='cuda:0') (tensor(9.8432, device='cuda:0')) + | > loss_gen: 1.8149049282073975 (1.929949329745385) + | > loss_kl: 2.3903613090515137 (2.5772578639368855) + | > loss_feat: 2.827540159225464 (3.020835399627685) + | > loss_mel: 24.418764114379883 (24.25547827443769) + | > loss_duration: 1.4658170938491821 (1.4014142790148336) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.917388916015625 (33.184934923725734) + | > grad_norm_1: tensor(133.3411, device='cuda:0') (tensor(166.5435, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.7791 (0.7819033668887231) + | > loader_time: 0.0103 (0.009143237144716324) + + + --> TIME: 2024-06-25 04:18:05 -- STEP: 56/591 -- GLOBAL_STEP: 5375 + | > loss_disc: 2.7299106121063232 (2.7568855285644536) + | > loss_disc_real_0: 0.2023763209581375 (0.22105984443000384) + | > loss_disc_real_1: 0.27242717146873474 (0.23681574128568172) + | > loss_disc_real_2: 0.19976554811000824 (0.2352586444467306) + | > loss_disc_real_3: 0.17709414660930634 (0.23766769974359445) + | > loss_disc_real_4: 0.18311041593551636 (0.2378467536930527) + | > loss_disc_real_5: 0.21321403980255127 (0.2391364428081683) + | > loss_0: 2.7299106121063232 (2.7568855285644536) + | > grad_norm_0: tensor(6.7264, device='cuda:0') (tensor(10.3405, device='cuda:0')) + | > loss_gen: 2.149746894836426 (1.903677184666906) + | > loss_kl: 2.4633750915527344 (2.520132554428918) + | > loss_feat: 3.027292490005493 (2.9306772521563937) + | > loss_mel: 24.370100021362305 (24.21014564377921) + | > loss_duration: 1.4734845161437988 (1.4164835001741136) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.4839973449707 (32.981115954262854) + | > grad_norm_1: tensor(221.4475, device='cuda:0') (tensor(146.9355, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.8034 (0.7931114605494908) + | > loader_time: 0.0099 (0.009682795831135344) + + + --> TIME: 2024-06-25 04:18:26 -- STEP: 81/591 -- GLOBAL_STEP: 5400 + | > loss_disc: 2.7285218238830566 (2.7537336849871976) + | > loss_disc_real_0: 0.25777825713157654 (0.2230456836613608) + | > loss_disc_real_1: 0.24839982390403748 (0.2335286204829628) + | > loss_disc_real_2: 0.23651103675365448 (0.235722420576178) + | > loss_disc_real_3: 0.20072980225086212 (0.23701426055696276) + | > loss_disc_real_4: 0.2201399952173233 (0.23779993973396443) + | > loss_disc_real_5: 0.24727775156497955 (0.2387754092981786) + | > loss_0: 2.7285218238830566 (2.7537336849871976) + | > grad_norm_0: tensor(7.3753, device='cuda:0') (tensor(10.5107, device='cuda:0')) + | > loss_gen: 1.8998558521270752 (1.890255673432056) + | > loss_kl: 2.422257900238037 (2.494452764958511) + | > loss_feat: 2.8940911293029785 (2.9045396469257496) + | > loss_mel: 23.92011260986328 (24.167996324138876) + | > loss_duration: 1.4485701322555542 (1.4246698765107144) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.58488845825195 (32.88191411524642) + | > grad_norm_1: tensor(186.4977, device='cuda:0') (tensor(139.5796, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.8172 (0.8012276461094986) + | > loader_time: 0.0114 (0.010134417333720643) + + + --> TIME: 2024-06-25 04:18:47 -- STEP: 106/591 -- GLOBAL_STEP: 5425 + | > loss_disc: 2.7085928916931152 (2.7515220012304917) + | > loss_disc_real_0: 0.18681132793426514 (0.2223806450811197) + | > loss_disc_real_1: 0.18776606023311615 (0.23125362944490505) + | > loss_disc_real_2: 0.24272526800632477 (0.2358192816938994) + | > loss_disc_real_3: 0.2426988184452057 (0.2370925372782743) + | > loss_disc_real_4: 0.23904471099376678 (0.23819056040835832) + | > loss_disc_real_5: 0.2543104290962219 (0.23946762478576517) + | > loss_0: 2.7085928916931152 (2.7515220012304917) + | > grad_norm_0: tensor(6.9893, device='cuda:0') (tensor(10.0622, device='cuda:0')) + | > loss_gen: 1.8661764860153198 (1.885903776816602) + | > loss_kl: 2.3623785972595215 (2.4752040291732214) + | > loss_feat: 2.7834103107452393 (2.9028801985506743) + | > loss_mel: 23.721033096313477 (24.123563784473347) + | > loss_duration: 1.43865966796875 (1.4305993509742452) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.17165756225586 (32.818150952177234) + | > grad_norm_1: tensor(109.7237, device='cuda:0') (tensor(135.8149, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.8365 (0.8091873245419197) + | > loader_time: 0.0135 (0.010474490669538388) + + + --> TIME: 2024-06-25 04:19:09 -- STEP: 131/591 -- GLOBAL_STEP: 5450 + | > loss_disc: 2.7737202644348145 (2.750073023424804) + | > loss_disc_real_0: 0.24840053915977478 (0.22419929268364688) + | > loss_disc_real_1: 0.2277805507183075 (0.2308089330451179) + | > loss_disc_real_2: 0.2572551369667053 (0.23533401416458247) + | > loss_disc_real_3: 0.19569799304008484 (0.23660085901959252) + | > loss_disc_real_4: 0.20222264528274536 (0.2370007578425735) + | > loss_disc_real_5: 0.1971309781074524 (0.238469424366041) + | > loss_0: 2.7737202644348145 (2.750073023424804) + | > grad_norm_0: tensor(6.2329, device='cuda:0') (tensor(10.6067, device='cuda:0')) + | > loss_gen: 1.8139548301696777 (1.889719589975954) + | > loss_kl: 2.405040979385376 (2.466591602063361) + | > loss_feat: 2.767867088317871 (2.9144081796398598) + | > loss_mel: 23.937406539916992 (24.1128645161636) + | > loss_duration: 1.4685142040252686 (1.432211778546108) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.392784118652344 (32.81579547620003) + | > grad_norm_1: tensor(163.1341, device='cuda:0') (tensor(139.6633, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.8523 (0.8170416518932081) + | > loader_time: 0.0116 (0.010709018197678426) + + + --> TIME: 2024-06-25 04:19:30 -- STEP: 156/591 -- GLOBAL_STEP: 5475 + | > loss_disc: 2.81188702583313 (2.747237726663932) + | > loss_disc_real_0: 0.3391473889350891 (0.22309336297882673) + | > loss_disc_real_1: 0.27039584517478943 (0.2301413886822187) + | > loss_disc_real_2: 0.22635728120803833 (0.235928226262331) + | > loss_disc_real_3: 0.21995319426059723 (0.23699170876389894) + | > loss_disc_real_4: 0.19569504261016846 (0.23730664260876486) + | > loss_disc_real_5: 0.2220071703195572 (0.23840952024627954) + | > loss_0: 2.81188702583313 (2.747237726663932) + | > grad_norm_0: tensor(22.4436, device='cuda:0') (tensor(10.2718, device='cuda:0')) + | > loss_gen: 1.6827003955841064 (1.8910474494481697) + | > loss_kl: 2.5283849239349365 (2.461095183323591) + | > loss_feat: 2.753488302230835 (2.917108320272886) + | > loss_mel: 23.32546043395996 (24.08593392983461) + | > loss_duration: 1.422621250152588 (1.4307032502614543) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.712656021118164 (32.785887877146415) + | > grad_norm_1: tensor(50.0232, device='cuda:0') (tensor(137.7308, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.87 (0.8228252331415812) + | > loader_time: 0.0126 (0.010880344953292452) + + + --> TIME: 2024-06-25 04:19:52 -- STEP: 181/591 -- GLOBAL_STEP: 5500 + | > loss_disc: 2.679309368133545 (2.740607731908723) + | > loss_disc_real_0: 0.15746548771858215 (0.2218122085690169) + | > loss_disc_real_1: 0.24071338772773743 (0.2292498152558975) + | > loss_disc_real_2: 0.2126004695892334 (0.2353894595283171) + | > loss_disc_real_3: 0.2507093548774719 (0.2365985073108041) + | > loss_disc_real_4: 0.24555432796478271 (0.23676287996176199) + | > loss_disc_real_5: 0.22574634850025177 (0.23821024629621876) + | > loss_0: 2.679309368133545 (2.740607731908723) + | > grad_norm_0: tensor(14.4692, device='cuda:0') (tensor(10.1985, device='cuda:0')) + | > loss_gen: 1.9136337041854858 (1.8932001959553082) + | > loss_kl: 2.4622676372528076 (2.459811180336041) + | > loss_feat: 3.0037052631378174 (2.9270820604503474) + | > loss_mel: 24.257701873779297 (24.07100796567802) + | > loss_duration: 1.4214931726455688 (1.4286508079391829) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.058799743652344 (32.77975191990975) + | > grad_norm_1: tensor(111.8309, device='cuda:0') (tensor(138.6624, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.8583 (0.8282583286749066) + | > loader_time: 0.013 (0.01103289614724849) + + + --> TIME: 2024-06-25 04:20:14 -- STEP: 206/591 -- GLOBAL_STEP: 5525 + | > loss_disc: 2.6728899478912354 (2.7385429803607524) + | > loss_disc_real_0: 0.12777967751026154 (0.22192287779621128) + | > loss_disc_real_1: 0.18806418776512146 (0.22842966287749486) + | > loss_disc_real_2: 0.25756120681762695 (0.23537164311675193) + | > loss_disc_real_3: 0.17692899703979492 (0.23620985802805539) + | > loss_disc_real_4: 0.2412881851196289 (0.23648881565019922) + | > loss_disc_real_5: 0.2591291666030884 (0.23786547686000473) + | > loss_0: 2.6728899478912354 (2.7385429803607524) + | > grad_norm_0: tensor(17.6420, device='cuda:0') (tensor(10.3206, device='cuda:0')) + | > loss_gen: 2.1028130054473877 (1.8942807351501243) + | > loss_kl: 2.253000497817993 (2.45289122942582) + | > loss_feat: 3.1086578369140625 (2.9346410128676763) + | > loss_mel: 24.30394744873047 (24.069113009184317) + | > loss_duration: 1.4232360124588013 (1.4278347445923159) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.191654205322266 (32.778760456344465) + | > grad_norm_1: tensor(87.9607, device='cuda:0') (tensor(137.0045, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.8951 (0.8336673789811366) + | > loader_time: 0.0136 (0.01124460141635635) + + + --> TIME: 2024-06-25 04:20:37 -- STEP: 231/591 -- GLOBAL_STEP: 5550 + | > loss_disc: 2.7510323524475098 (2.736702045837005) + | > loss_disc_real_0: 0.20132121443748474 (0.22188140963127603) + | > loss_disc_real_1: 0.21036645770072937 (0.22812267389648405) + | > loss_disc_real_2: 0.24052178859710693 (0.23534480492015938) + | > loss_disc_real_3: 0.236558735370636 (0.2364733721012677) + | > loss_disc_real_4: 0.24337732791900635 (0.2364236165434767) + | > loss_disc_real_5: 0.23109646141529083 (0.2377114219840987) + | > loss_0: 2.7510323524475098 (2.736702045837005) + | > grad_norm_0: tensor(4.8422, device='cuda:0') (tensor(10.1252, device='cuda:0')) + | > loss_gen: 1.8417123556137085 (1.894706569708787) + | > loss_kl: 2.450978994369507 (2.4510789454241344) + | > loss_feat: 2.852304220199585 (2.938595370296791) + | > loss_mel: 24.02737045288086 (24.073594510297248) + | > loss_duration: 1.4372084140777588 (1.4276995065408362) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.60957717895508 (32.7856746772667) + | > grad_norm_1: tensor(91.4289, device='cuda:0') (tensor(132.2172, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.8765 (0.8391652912288518) + | > loader_time: 0.0137 (0.011528022361524174) + + + --> TIME: 2024-06-25 04:21:00 -- STEP: 256/591 -- GLOBAL_STEP: 5575 + | > loss_disc: 2.665998935699463 (2.7357322666794053) + | > loss_disc_real_0: 0.20642463862895966 (0.22185459367756266) + | > loss_disc_real_1: 0.2548835575580597 (0.22779738868121058) + | > loss_disc_real_2: 0.24116894602775574 (0.23540174728259444) + | > loss_disc_real_3: 0.2627684473991394 (0.2364732656860724) + | > loss_disc_real_4: 0.23641806840896606 (0.2362477844581008) + | > loss_disc_real_5: 0.23933246731758118 (0.2377713765599765) + | > loss_0: 2.665998935699463 (2.7357322666794053) + | > grad_norm_0: tensor(5.5294, device='cuda:0') (tensor(10.2412, device='cuda:0')) + | > loss_gen: 1.9533793926239014 (1.8961879690177739) + | > loss_kl: 2.3780593872070312 (2.450412457808853) + | > loss_feat: 3.0661466121673584 (2.93917485885322) + | > loss_mel: 23.308135986328125 (24.06048375368119) + | > loss_duration: 1.451398253440857 (1.4277875078842053) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.15711975097656 (32.77404634654523) + | > grad_norm_1: tensor(80.8969, device='cuda:0') (tensor(131.2738, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.9047 (0.8446282064542174) + | > loader_time: 0.0129 (0.011823410168290135) + + + --> TIME: 2024-06-25 04:21:23 -- STEP: 281/591 -- GLOBAL_STEP: 5600 + | > loss_disc: 2.6449456214904785 (2.7355694253249516) + | > loss_disc_real_0: 0.21211116015911102 (0.22186064223006527) + | > loss_disc_real_1: 0.17968863248825073 (0.22754994734750525) + | > loss_disc_real_2: 0.19243232905864716 (0.2351477251676478) + | > loss_disc_real_3: 0.21322323381900787 (0.236177571780741) + | > loss_disc_real_4: 0.21819891035556793 (0.236214227576697) + | > loss_disc_real_5: 0.27426356077194214 (0.23783373636495175) + | > loss_0: 2.6449456214904785 (2.7355694253249516) + | > grad_norm_0: tensor(4.8538, device='cuda:0') (tensor(10.4013, device='cuda:0')) + | > loss_gen: 2.0586740970611572 (1.895845984224747) + | > loss_kl: 2.3520989418029785 (2.448656902618681) + | > loss_feat: 3.1232666969299316 (2.938602013095841) + | > loss_mel: 24.131603240966797 (24.029944626886227) + | > loss_duration: 1.463205099105835 (1.4285735960957002) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.12884521484375 (32.74162291801698) + | > grad_norm_1: tensor(218.9974, device='cuda:0') (tensor(131.9096, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.9136 (0.8497356757574657) + | > loader_time: 0.0161 (0.012117586950390362) + + + --> TIME: 2024-06-25 04:21:46 -- STEP: 306/591 -- GLOBAL_STEP: 5625 + | > loss_disc: 2.77191162109375 (2.7362797945932615) + | > loss_disc_real_0: 0.2548244595527649 (0.22205129892754008) + | > loss_disc_real_1: 0.21780087053775787 (0.22785031849067974) + | > loss_disc_real_2: 0.22376643121242523 (0.23537898487320133) + | > loss_disc_real_3: 0.3143281638622284 (0.23655369244759378) + | > loss_disc_real_4: 0.23914381861686707 (0.23625435952851975) + | > loss_disc_real_5: 0.2460811585187912 (0.2378069001669977) + | > loss_0: 2.77191162109375 (2.7362797945932615) + | > grad_norm_0: tensor(7.3115, device='cuda:0') (tensor(10.3680, device='cuda:0')) + | > loss_gen: 1.7081375122070312 (1.894964457726946) + | > loss_kl: 2.397407293319702 (2.4450098798165927) + | > loss_feat: 2.634640693664551 (2.933000902724421) + | > loss_mel: 24.70948028564453 (24.01484033796522) + | > loss_duration: 1.473312258720398 (1.429016130422456) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.922977447509766 (32.716831500234164) + | > grad_norm_1: tensor(157.2984, device='cuda:0') (tensor(130.2640, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.9255 (0.8555052763496346) + | > loader_time: 0.0156 (0.012343811833001421) + + + --> TIME: 2024-06-25 04:22:10 -- STEP: 331/591 -- GLOBAL_STEP: 5650 + | > loss_disc: 2.65848970413208 (2.7341962746623047) + | > loss_disc_real_0: 0.250216007232666 (0.22183352300272607) + | > loss_disc_real_1: 0.1976589560508728 (0.22733385806897616) + | > loss_disc_real_2: 0.19995121657848358 (0.23486567160334113) + | > loss_disc_real_3: 0.21759866178035736 (0.23652096299786582) + | > loss_disc_real_4: 0.23047327995300293 (0.235994611441909) + | > loss_disc_real_5: 0.2187284231185913 (0.2374473317242101) + | > loss_0: 2.65848970413208 (2.7341962746623047) + | > grad_norm_0: tensor(9.4495, device='cuda:0') (tensor(10.5236, device='cuda:0')) + | > loss_gen: 2.1063649654388428 (1.8965904370535536) + | > loss_kl: 2.3761696815490723 (2.443031579706245) + | > loss_feat: 3.0630412101745605 (2.937585313514638) + | > loss_mel: 23.119531631469727 (24.015324232441422) + | > loss_duration: 1.4187822341918945 (1.429534726272538) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.08388900756836 (32.72206610711317) + | > grad_norm_1: tensor(96.2047, device='cuda:0') (tensor(131.6006, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.9398 (0.8605014752045135) + | > loader_time: 0.0145 (0.012574357208528536) + + + --> TIME: 2024-06-25 04:22:34 -- STEP: 356/591 -- GLOBAL_STEP: 5675 + | > loss_disc: 2.6808416843414307 (2.7340379866321425) + | > loss_disc_real_0: 0.22146308422088623 (0.22179356643281292) + | > loss_disc_real_1: 0.2743796408176422 (0.2274476546752319) + | > loss_disc_real_2: 0.22703227400779724 (0.2350138992573438) + | > loss_disc_real_3: 0.17525333166122437 (0.23653185535013005) + | > loss_disc_real_4: 0.21961304545402527 (0.23584102081616273) + | > loss_disc_real_5: 0.20651616156101227 (0.2372597564388527) + | > loss_0: 2.6808416843414307 (2.7340379866321425) + | > grad_norm_0: tensor(5.8644, device='cuda:0') (tensor(10.6382, device='cuda:0')) + | > loss_gen: 1.9201580286026 (1.8960186226314373) + | > loss_kl: 2.4042680263519287 (2.4405713188514286) + | > loss_feat: 3.006483793258667 (2.9362594337945556) + | > loss_mel: 24.20490264892578 (24.004362106323242) + | > loss_duration: 1.4480838775634766 (1.430687669622765) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.98389434814453 (32.70789896504261) + | > grad_norm_1: tensor(272.2069, device='cuda:0') (tensor(131.8993, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.9332 (0.8659603006384343) + | > loader_time: 0.0173 (0.012841243422433223) + + + --> TIME: 2024-06-25 04:22:59 -- STEP: 381/591 -- GLOBAL_STEP: 5700 + | > loss_disc: 2.6764798164367676 (2.7334669168226995) + | > loss_disc_real_0: 0.20871764421463013 (0.22139677537355837) + | > loss_disc_real_1: 0.22387635707855225 (0.22742222163464454) + | > loss_disc_real_2: 0.22920137643814087 (0.23506171891852) + | > loss_disc_real_3: 0.19580461084842682 (0.23636514656343485) + | > loss_disc_real_4: 0.2091982215642929 (0.23594896245034036) + | > loss_disc_real_5: 0.21124589443206787 (0.23734092086631794) + | > loss_0: 2.6764798164367676 (2.7334669168226995) + | > grad_norm_0: tensor(4.8592, device='cuda:0') (tensor(10.5215, device='cuda:0')) + | > loss_gen: 1.9122509956359863 (1.8961071727156953) + | > loss_kl: 2.3592493534088135 (2.437944163800539) + | > loss_feat: 2.9808390140533447 (2.933698438909734) + | > loss_mel: 23.360889434814453 (23.98306462145227) + | > loss_duration: 1.4620192050933838 (1.4323741228874598) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.07524871826172 (32.68318836081996) + | > grad_norm_1: tensor(52.5835, device='cuda:0') (tensor(131.2625, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 0.9732 (0.8724314916165166) + | > loader_time: 0.0164 (0.013061737451027688) + + + --> TIME: 2024-06-25 04:23:24 -- STEP: 406/591 -- GLOBAL_STEP: 5725 + | > loss_disc: 2.708765983581543 (2.732429971835884) + | > loss_disc_real_0: 0.23556962609291077 (0.22125429284800155) + | > loss_disc_real_1: 0.2197444885969162 (0.22714542527797774) + | > loss_disc_real_2: 0.23881089687347412 (0.23496677128258597) + | > loss_disc_real_3: 0.2629729211330414 (0.23640919906165211) + | > loss_disc_real_4: 0.22769643366336823 (0.23590945522186205) + | > loss_disc_real_5: 0.2835872769355774 (0.2373742038026232) + | > loss_0: 2.708765983581543 (2.732429971835884) + | > grad_norm_0: tensor(4.6275, device='cuda:0') (tensor(10.5456, device='cuda:0')) + | > loss_gen: 1.927727222442627 (1.8963438947799758) + | > loss_kl: 2.4739809036254883 (2.4368866071325233) + | > loss_feat: 2.957975149154663 (2.9306932576184184) + | > loss_mel: 23.444673538208008 (23.960012558058565) + | > loss_duration: 1.4218698740005493 (1.4331658732127686) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.226226806640625 (32.65710205867369) + | > grad_norm_1: tensor(55.2052, device='cuda:0') (tensor(130.8535, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 1.0118 (0.8787250853524416) + | > loader_time: 0.0166 (0.013315600714660035) + + + --> TIME: 2024-06-25 04:23:49 -- STEP: 431/591 -- GLOBAL_STEP: 5750 + | > loss_disc: 2.7275478839874268 (2.7321072277502787) + | > loss_disc_real_0: 0.1586034595966339 (0.22118868081551693) + | > loss_disc_real_1: 0.16377539932727814 (0.22690263186557663) + | > loss_disc_real_2: 0.22246092557907104 (0.2349514304001204) + | > loss_disc_real_3: 0.21311040222644806 (0.2363077608172689) + | > loss_disc_real_4: 0.22886048257350922 (0.23584404262755143) + | > loss_disc_real_5: 0.24909168481826782 (0.23738006680443227) + | > loss_0: 2.7275478839874268 (2.7321072277502787) + | > grad_norm_0: tensor(13.0632, device='cuda:0') (tensor(10.5519, device='cuda:0')) + | > loss_gen: 2.105278253555298 (1.896979057871825) + | > loss_kl: 2.45455265045166 (2.4361352765643143) + | > loss_feat: 2.8948822021484375 (2.929348847705636) + | > loss_mel: 23.21232795715332 (23.946017710072812) + | > loss_duration: 1.4484152793884277 (1.4337986278976487) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.115455627441406 (32.642279366052854) + | > grad_norm_1: tensor(187.2159, device='cuda:0') (tensor(129.0061, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 1.0028 (0.8856137368629258) + | > loader_time: 0.0179 (0.013588710616746793) + + + --> TIME: 2024-06-25 04:24:15 -- STEP: 456/591 -- GLOBAL_STEP: 5775 + | > loss_disc: 2.696195125579834 (2.7324471159985197) + | > loss_disc_real_0: 0.11149947345256805 (0.2212842445608163) + | > loss_disc_real_1: 0.22975319623947144 (0.2272170252705875) + | > loss_disc_real_2: 0.2171173244714737 (0.23485489215767175) + | > loss_disc_real_3: 0.26070860028266907 (0.23631634651438185) + | > loss_disc_real_4: 0.20279988646507263 (0.23580929823219776) + | > loss_disc_real_5: 0.21795085072517395 (0.23730662719983803) + | > loss_0: 2.696195125579834 (2.7324471159985197) + | > grad_norm_0: tensor(22.6140, device='cuda:0') (tensor(10.5903, device='cuda:0')) + | > loss_gen: 2.0752151012420654 (1.8969616557945286) + | > loss_kl: 2.3084750175476074 (2.4332840071435573) + | > loss_feat: 2.9743008613586426 (2.9241658877908123) + | > loss_mel: 23.09091567993164 (23.92700452971877) + | > loss_duration: 1.4327555894851685 (1.434649578573412) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.881662368774414 (32.61606551471507) + | > grad_norm_1: tensor(186.7707, device='cuda:0') (tensor(129.6466, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 1.0112 (0.8927752427887495) + | > loader_time: 0.02 (0.013889800038254045) + + + --> TIME: 2024-06-25 04:24:41 -- STEP: 481/591 -- GLOBAL_STEP: 5800 + | > loss_disc: 2.717088222503662 (2.7319142560701115) + | > loss_disc_real_0: 0.19584967195987701 (0.22122213163116866) + | > loss_disc_real_1: 0.18387232720851898 (0.22709450869203357) + | > loss_disc_real_2: 0.20108537375926971 (0.23479878806398713) + | > loss_disc_real_3: 0.3055393099784851 (0.23636460505627296) + | > loss_disc_real_4: 0.2625528573989868 (0.23595085241318742) + | > loss_disc_real_5: 0.2595113515853882 (0.23744409384202064) + | > loss_0: 2.717088222503662 (2.7319142560701115) + | > grad_norm_0: tensor(7.1473, device='cuda:0') (tensor(10.5878, device='cuda:0')) + | > loss_gen: 1.8432841300964355 (1.8972030404699567) + | > loss_kl: 2.335155963897705 (2.431040987899052) + | > loss_feat: 2.8616063594818115 (2.9207073859018484) + | > loss_mel: 22.729572296142578 (23.90855996574037) + | > loss_duration: 1.4226019382476807 (1.4358934082261252) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.19222068786621 (32.59340466679748) + | > grad_norm_1: tensor(78.6735, device='cuda:0') (tensor(130.6254, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 1.0736 (0.8999033980458787) + | > loader_time: 0.0192 (0.01417542049146243) + + + --> TIME: 2024-06-25 04:25:09 -- STEP: 506/591 -- GLOBAL_STEP: 5825 + | > loss_disc: 2.76503586769104 (2.731589565635199) + | > loss_disc_real_0: 0.35343143343925476 (0.22134323826744387) + | > loss_disc_real_1: 0.15432511270046234 (0.2268864159645299) + | > loss_disc_real_2: 0.23612378537654877 (0.23473777719165967) + | > loss_disc_real_3: 0.22347742319107056 (0.23629649663748947) + | > loss_disc_real_4: 0.21369630098342896 (0.23589324792147626) + | > loss_disc_real_5: 0.225799098610878 (0.2374354332921062) + | > loss_0: 2.76503586769104 (2.731589565635199) + | > grad_norm_0: tensor(19.1061, device='cuda:0') (tensor(10.6994, device='cuda:0')) + | > loss_gen: 1.7448323965072632 (1.897050522061676) + | > loss_kl: 2.3717591762542725 (2.4286619971392196) + | > loss_feat: 2.8265867233276367 (2.918008963109946) + | > loss_mel: 23.39098358154297 (23.884769594245277) + | > loss_duration: 1.459367275238037 (1.4372826371739513) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.793529510498047 (32.565773602059664) + | > grad_norm_1: tensor(192.4022, device='cuda:0') (tensor(131.0145, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 1.1899 (0.9087580456564075) + | > loader_time: 0.023 (0.014474388638975114) + + + --> TIME: 2024-06-25 04:25:38 -- STEP: 531/591 -- GLOBAL_STEP: 5850 + | > loss_disc: 2.763632297515869 (2.730205672384431) + | > loss_disc_real_0: 0.18715806305408478 (0.22097528347720982) + | > loss_disc_real_1: 0.24860453605651855 (0.22690523880237465) + | > loss_disc_real_2: 0.26234444975852966 (0.23458977414053037) + | > loss_disc_real_3: 0.26711881160736084 (0.23623784416812962) + | > loss_disc_real_4: 0.2062181979417801 (0.23577948571261714) + | > loss_disc_real_5: 0.232330322265625 (0.23741634987056143) + | > loss_0: 2.763632297515869 (2.730205672384431) + | > grad_norm_0: tensor(6.8866, device='cuda:0') (tensor(10.6277, device='cuda:0')) + | > loss_gen: 1.8248496055603027 (1.897672227097994) + | > loss_kl: 2.369459629058838 (2.42604404474607) + | > loss_feat: 2.8508999347686768 (2.9176829058796216) + | > loss_mel: 23.117900848388672 (23.87306108133492) + | > loss_duration: 1.4917203187942505 (1.4387317287495367) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.654830932617188 (32.5531918728643) + | > grad_norm_1: tensor(63.2188, device='cuda:0') (tensor(130.2068, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 1.18 (0.9191502617801872) + | > loader_time: 0.0227 (0.014797590995687991) + + + --> TIME: 2024-06-25 04:26:09 -- STEP: 556/591 -- GLOBAL_STEP: 5875 + | > loss_disc: 2.661978006362915 (2.7308478689879827) + | > loss_disc_real_0: 0.17977061867713928 (0.22119250406491034) + | > loss_disc_real_1: 0.2755207419395447 (0.22702512606013592) + | > loss_disc_real_2: 0.20986196398735046 (0.23447903044682614) + | > loss_disc_real_3: 0.28851500153541565 (0.2361592835069989) + | > loss_disc_real_4: 0.17713236808776855 (0.2357430248755774) + | > loss_disc_real_5: 0.22481003403663635 (0.23745508516113534) + | > loss_0: 2.661978006362915 (2.7308478689879827) + | > grad_norm_0: tensor(6.6611, device='cuda:0') (tensor(10.7807, device='cuda:0')) + | > loss_gen: 2.000354528427124 (1.8978004785750409) + | > loss_kl: 2.332348585128784 (2.4229533007676665) + | > loss_feat: 3.018084764480591 (2.912397873487404) + | > loss_mel: 23.492706298828125 (23.83672711832061) + | > loss_duration: 1.4936540126800537 (1.4402912857292378) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.3371467590332 (32.51016993831384) + | > grad_norm_1: tensor(68.9561, device='cuda:0') (tensor(130.1796, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 1.204 (0.9320909994111645) + | > loader_time: 0.0261 (0.01513491045656821) + + + --> TIME: 2024-06-25 04:26:41 -- STEP: 581/591 -- GLOBAL_STEP: 5900 + | > loss_disc: 2.681572198867798 (2.729748083679278) + | > loss_disc_real_0: 0.20703160762786865 (0.22086021135498662) + | > loss_disc_real_1: 0.2104642242193222 (0.2270184810467308) + | > loss_disc_real_2: 0.20917733013629913 (0.23444477346903692) + | > loss_disc_real_3: 0.24451276659965515 (0.23607816817735847) + | > loss_disc_real_4: 0.23900236189365387 (0.23577069936131037) + | > loss_disc_real_5: 0.22518590092658997 (0.23747056286634555) + | > loss_0: 2.681572198867798 (2.729748083679278) + | > grad_norm_0: tensor(6.1729, device='cuda:0') (tensor(10.6511, device='cuda:0')) + | > loss_gen: 1.7496471405029297 (1.8974815929202737) + | > loss_kl: 2.3675267696380615 (2.4208430501146707) + | > loss_feat: 2.8783459663391113 (2.90900911644691) + | > loss_mel: 23.540815353393555 (23.820829696786596) + | > loss_duration: 1.4754928350448608 (1.4417477536324592) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.0118293762207 (32.48991107776526) + | > grad_norm_1: tensor(195.9055, device='cuda:0') (tensor(131.3510, device='cuda:0')) + | > current_lr_0: 0.0001997751124671936 + | > current_lr_1: 0.0001997751124671936 + | > step_time: 1.3459 (0.946376081177782) + | > loader_time: 0.0267 (0.015569422208093318) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035835732804968003 (-9.280062736348894e-05) + | > avg_loss_disc: 2.7563347258466355 (+0.032579914052435655) + | > avg_loss_disc_real_0: 0.2535686889227401 (+0.002965131655652542) + | > avg_loss_disc_real_1: 0.19817308225530259 (-0.06362542066168272) + | > avg_loss_disc_real_2: 0.2582502669476449 (-0.00025620676101517637) + | > avg_loss_disc_real_3: 0.21088573907284028 (-0.08802819886106125) + | > avg_loss_disc_real_4: 0.2247399795562663 (+0.006936917279629001) + | > avg_loss_disc_real_5: 0.2276284688330711 (-0.00580662362118986) + | > avg_loss_0: 2.7563347258466355 (+0.032579914052435655) + | > avg_loss_gen: 1.8100691983040342 (-0.22331936055041512) + | > avg_loss_kl: 2.443449786368836 (+0.003295416527605788) + | > avg_loss_feat: 2.7958069050565686 (-0.06899220639086723) + | > avg_loss_mel: 23.694090660582198 (-0.1640285329615807) + | > avg_loss_duration: 1.4253752282325258 (+0.0004912589458709871) + | > avg_loss_1: 32.16879219704485 (-0.4525529983195966) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_5910.pth + + > EPOCH: 11/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 04:27:06)  + + --> TIME: 2024-06-25 04:27:20 -- STEP: 15/591 -- GLOBAL_STEP: 5925 + | > loss_disc: 2.767179489135742 (2.7546116511027017) + | > loss_disc_real_0: 0.24251464009284973 (0.22502183218797048) + | > loss_disc_real_1: 0.2209274172782898 (0.22923943996429444) + | > loss_disc_real_2: 0.2264142781496048 (0.23505894442399342) + | > loss_disc_real_3: 0.23415006697177887 (0.24205385148525238) + | > loss_disc_real_4: 0.2133924812078476 (0.24034389456113178) + | > loss_disc_real_5: 0.26265451312065125 (0.24196296334266662) + | > loss_0: 2.767179489135742 (2.7546116511027017) + | > grad_norm_0: tensor(7.4552, device='cuda:0') (tensor(13.6177, device='cuda:0')) + | > loss_gen: 1.8318367004394531 (1.892301909128825) + | > loss_kl: 2.505058526992798 (2.659201669692993) + | > loss_feat: 3.019002676010132 (3.022876437505086) + | > loss_mel: 23.258743286132812 (23.717566935221353) + | > loss_duration: 1.411159873008728 (1.3921451091766357) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.02580261230469 (32.68409194946289) + | > grad_norm_1: tensor(107.6584, device='cuda:0') (tensor(90.5448, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.7832 (0.7724988619486491) + | > loader_time: 0.0092 (0.00852352778116862) + + + --> TIME: 2024-06-25 04:27:40 -- STEP: 40/591 -- GLOBAL_STEP: 5950 + | > loss_disc: 2.7267026901245117 (2.7456371426582336) + | > loss_disc_real_0: 0.2022821009159088 (0.2189764130860567) + | > loss_disc_real_1: 0.2800801992416382 (0.22913947477936744) + | > loss_disc_real_2: 0.22991156578063965 (0.23585076369345187) + | > loss_disc_real_3: 0.2475453019142151 (0.2382361028343439) + | > loss_disc_real_4: 0.24427102506160736 (0.23751841560006143) + | > loss_disc_real_5: 0.24724438786506653 (0.23883342780172825) + | > loss_0: 2.7267026901245117 (2.7456371426582336) + | > grad_norm_0: tensor(4.6409, device='cuda:0') (tensor(11.1957, device='cuda:0')) + | > loss_gen: 2.0111958980560303 (1.8956935584545136) + | > loss_kl: 2.4945592880249023 (2.568796294927597) + | > loss_feat: 2.993044376373291 (2.9924312472343444) + | > loss_mel: 24.599096298217773 (23.70339455604553) + | > loss_duration: 1.4402809143066406 (1.4010640174150466) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.538177490234375 (32.56137971878052) + | > grad_norm_1: tensor(260.1543, device='cuda:0') (tensor(122.9381, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.7989 (0.7846760988235474) + | > loader_time: 0.0104 (0.009195303916931153) + + + --> TIME: 2024-06-25 04:28:01 -- STEP: 65/591 -- GLOBAL_STEP: 5975 + | > loss_disc: 2.741240978240967 (2.746039390563965) + | > loss_disc_real_0: 0.31039971113204956 (0.2205676800929583) + | > loss_disc_real_1: 0.25921308994293213 (0.22880657154780168) + | > loss_disc_real_2: 0.24218197166919708 (0.23567162156105043) + | > loss_disc_real_3: 0.2676747143268585 (0.2388776827317018) + | > loss_disc_real_4: 0.19733472168445587 (0.2365648111471763) + | > loss_disc_real_5: 0.18966999650001526 (0.23933470661823567) + | > loss_0: 2.741240978240967 (2.746039390563965) + | > grad_norm_0: tensor(17.0467, device='cuda:0') (tensor(11.1684, device='cuda:0')) + | > loss_gen: 1.8427441120147705 (1.8917840590843789) + | > loss_kl: 2.4674761295318604 (2.5271492994748628) + | > loss_feat: 2.958620309829712 (2.9771574277144213) + | > loss_mel: 23.787857055664062 (23.690704345703125) + | > loss_duration: 1.4341872930526733 (1.410484295624953) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.49088668823242 (32.49727947528549) + | > grad_norm_1: tensor(172.5446, device='cuda:0') (tensor(141.1213, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.817 (0.7976054778465858) + | > loader_time: 0.0103 (0.009626164803138142) + + + --> TIME: 2024-06-25 04:28:22 -- STEP: 90/591 -- GLOBAL_STEP: 6000 + | > loss_disc: 2.7830047607421875 (2.7454610877566856) + | > loss_disc_real_0: 0.21360349655151367 (0.2204839375283983) + | > loss_disc_real_1: 0.31113654375076294 (0.2291163409749667) + | > loss_disc_real_2: 0.2235322743654251 (0.23510273728105757) + | > loss_disc_real_3: 0.24846376478672028 (0.2379991340968344) + | > loss_disc_real_4: 0.2460213154554367 (0.23597031682729722) + | > loss_disc_real_5: 0.2428414672613144 (0.2388974434799618) + | > loss_0: 2.7830047607421875 (2.7454610877566856) + | > grad_norm_0: tensor(5.0870, device='cuda:0') (tensor(11.1235, device='cuda:0')) + | > loss_gen: 1.9895195960998535 (1.8909935699568854) + | > loss_kl: 2.412907600402832 (2.505517721176148) + | > loss_feat: 3.0142314434051514 (2.975425272517733) + | > loss_mel: 24.270706176757812 (23.7313563240899) + | > loss_duration: 1.4510903358459473 (1.4183655977249143) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.13845443725586 (32.521658452351915) + | > grad_norm_1: tensor(207.2757, device='cuda:0') (tensor(146.4117, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.8282 (0.8050346242056954) + | > loader_time: 0.011 (0.010082080629136823) + + + --> TIME: 2024-06-25 04:28:43 -- STEP: 115/591 -- GLOBAL_STEP: 6025 + | > loss_disc: 2.6974713802337646 (2.7457527160644517) + | > loss_disc_real_0: 0.1415661722421646 (0.2202266777339189) + | > loss_disc_real_1: 0.21359261870384216 (0.22882922265840613) + | > loss_disc_real_2: 0.2034101039171219 (0.23629313940587252) + | > loss_disc_real_3: 0.2444952130317688 (0.23771045583745706) + | > loss_disc_real_4: 0.2207871973514557 (0.23637507622656614) + | > loss_disc_real_5: 0.21421289443969727 (0.23895081514897554) + | > loss_0: 2.6974713802337646 (2.7457527160644517) + | > grad_norm_0: tensor(16.1362, device='cuda:0') (tensor(10.6637, device='cuda:0')) + | > loss_gen: 1.972085952758789 (1.8878835532976233) + | > loss_kl: 2.435127019882202 (2.487670527333799) + | > loss_feat: 3.0515193939208984 (2.9693682173024043) + | > loss_mel: 23.325937271118164 (23.732873634670096) + | > loss_duration: 1.4296399354934692 (1.4217838401379792) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.21430969238281 (32.49957981938903) + | > grad_norm_1: tensor(210.9605, device='cuda:0') (tensor(138.9844, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.8505 (0.8114108873450239) + | > loader_time: 0.0108 (0.010346555709838863) + + + --> TIME: 2024-06-25 04:29:05 -- STEP: 140/591 -- GLOBAL_STEP: 6050 + | > loss_disc: 2.828880548477173 (2.7475314004080627) + | > loss_disc_real_0: 0.20361682772636414 (0.22117715924978257) + | > loss_disc_real_1: 0.3067123591899872 (0.22943758187549454) + | > loss_disc_real_2: 0.23378537595272064 (0.23656565472483634) + | > loss_disc_real_3: 0.2548312544822693 (0.2373892605304718) + | > loss_disc_real_4: 0.27287060022354126 (0.23594781649964197) + | > loss_disc_real_5: 0.2608298063278198 (0.238822895607778) + | > loss_0: 2.828880548477173 (2.7475314004080627) + | > grad_norm_0: tensor(5.2853, device='cuda:0') (tensor(11.0179, device='cuda:0')) + | > loss_gen: 1.8301324844360352 (1.8866398905004773) + | > loss_kl: 2.4807112216949463 (2.483083600657327) + | > loss_feat: 2.55645751953125 (2.961920714378356) + | > loss_mel: 23.440670013427734 (23.73129133496966) + | > loss_duration: 1.4013066291809082 (1.422109463385173) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.709278106689453 (32.485045037950805) + | > grad_norm_1: tensor(174.0586, device='cuda:0') (tensor(138.1657, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.8478 (0.8179345897265845) + | > loader_time: 0.0114 (0.0105892539024353) + + + --> TIME: 2024-06-25 04:29:27 -- STEP: 165/591 -- GLOBAL_STEP: 6075 + | > loss_disc: 2.739013433456421 (2.7452375874374857) + | > loss_disc_real_0: 0.18641293048858643 (0.2203720178568002) + | > loss_disc_real_1: 0.22321468591690063 (0.22936519266981067) + | > loss_disc_real_2: 0.2090001255273819 (0.23604566500042423) + | > loss_disc_real_3: 0.16685932874679565 (0.23753933509190878) + | > loss_disc_real_4: 0.22230730950832367 (0.2354337341857679) + | > loss_disc_real_5: 0.1891998052597046 (0.2384059673908985) + | > loss_0: 2.739013433456421 (2.7452375874374857) + | > grad_norm_0: tensor(5.5210, device='cuda:0') (tensor(10.7598, device='cuda:0')) + | > loss_gen: 1.9631789922714233 (1.888057973168113) + | > loss_kl: 2.522608518600464 (2.479065289641873) + | > loss_feat: 3.0125529766082764 (2.96013432560545) + | > loss_mel: 23.587215423583984 (23.722058729691938) + | > loss_duration: 1.3916304111480713 (1.4204270189458676) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.47718811035156 (32.469743358727676) + | > grad_norm_1: tensor(59.9740, device='cuda:0') (tensor(136.2389, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.8579 (0.8242518569483905) + | > loader_time: 0.0114 (0.010757151517001061) + + + --> TIME: 2024-06-25 04:29:49 -- STEP: 190/591 -- GLOBAL_STEP: 6100 + | > loss_disc: 2.775343418121338 (2.7450919715981748) + | > loss_disc_real_0: 0.18773669004440308 (0.2200286738182369) + | > loss_disc_real_1: 0.2269366830587387 (0.22930899999643628) + | > loss_disc_real_2: 0.20319317281246185 (0.2361953651434497) + | > loss_disc_real_3: 0.20327968895435333 (0.23745545086107755) + | > loss_disc_real_4: 0.2350529581308365 (0.2356355408304616) + | > loss_disc_real_5: 0.22302871942520142 (0.2381439463088387) + | > loss_0: 2.775343418121338 (2.7450919715981748) + | > grad_norm_0: tensor(7.7109, device='cuda:0') (tensor(10.8502, device='cuda:0')) + | > loss_gen: 1.9614286422729492 (1.8878165357991268) + | > loss_kl: 2.589833974838257 (2.4765835310283477) + | > loss_feat: 2.8629097938537598 (2.9520909070968626) + | > loss_mel: 23.253997802734375 (23.699092282746967) + | > loss_duration: 1.4493242502212524 (1.4192682385444644) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.117496490478516 (32.43485153599791) + | > grad_norm_1: tensor(152.9688, device='cuda:0') (tensor(136.3545, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.8725 (0.8298722882019849) + | > loader_time: 0.0115 (0.010968857062490359) + + + --> TIME: 2024-06-25 04:30:11 -- STEP: 215/591 -- GLOBAL_STEP: 6125 + | > loss_disc: 2.7916908264160156 (2.7442898994268385) + | > loss_disc_real_0: 0.12957963347434998 (0.21983042134102002) + | > loss_disc_real_1: 0.2754889130592346 (0.2296810431535854) + | > loss_disc_real_2: 0.26351597905158997 (0.23638740050238233) + | > loss_disc_real_3: 0.20693621039390564 (0.2371244867180669) + | > loss_disc_real_4: 0.26752784848213196 (0.235515790831211) + | > loss_disc_real_5: 0.2539147436618805 (0.23797985654930737) + | > loss_0: 2.7916908264160156 (2.7442898994268385) + | > grad_norm_0: tensor(18.6957, device='cuda:0') (tensor(10.8760, device='cuda:0')) + | > loss_gen: 2.0844013690948486 (1.888828388480253) + | > loss_kl: 2.3934412002563477 (2.4732748563899554) + | > loss_feat: 2.8453879356384277 (2.949994211418684) + | > loss_mel: 23.9504451751709 (23.696649276378544) + | > loss_duration: 1.441419243812561 (1.4185189535451492) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.71509552001953 (32.42726575274803) + | > grad_norm_1: tensor(213.9789, device='cuda:0') (tensor(137.8511, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.867 (0.8353154881055967) + | > loader_time: 0.013 (0.011264886412509648) + + + --> TIME: 2024-06-25 04:30:34 -- STEP: 240/591 -- GLOBAL_STEP: 6150 + | > loss_disc: 2.7617225646972656 (2.7425454477469136) + | > loss_disc_real_0: 0.08825305104255676 (0.2194740313105285) + | > loss_disc_real_1: 0.21665306389331818 (0.22896774150431157) + | > loss_disc_real_2: 0.21642924845218658 (0.23632557075470687) + | > loss_disc_real_3: 0.2785506546497345 (0.23705945536494255) + | > loss_disc_real_4: 0.2418850064277649 (0.23543348299960296) + | > loss_disc_real_5: 0.25308072566986084 (0.23795389973868927) + | > loss_0: 2.7617225646972656 (2.7425454477469136) + | > grad_norm_0: tensor(29.4419, device='cuda:0') (tensor(10.8929, device='cuda:0')) + | > loss_gen: 2.1444058418273926 (1.889552797873815) + | > loss_kl: 2.488285541534424 (2.4717939684788393) + | > loss_feat: 2.9495813846588135 (2.9506464739640554) + | > loss_mel: 23.900611877441406 (23.701224080721538) + | > loss_duration: 1.426492691040039 (1.4183770040671033) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.90937805175781 (32.431594355901105) + | > grad_norm_1: tensor(79.8437, device='cuda:0') (tensor(138.9936, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.8776 (0.8403075615564984) + | > loader_time: 0.0129 (0.011552520593007402) + + + --> TIME: 2024-06-25 04:30:57 -- STEP: 265/591 -- GLOBAL_STEP: 6175 + | > loss_disc: 2.873696804046631 (2.7437116146087654) + | > loss_disc_real_0: 0.4572001099586487 (0.2202507177332662) + | > loss_disc_real_1: 0.2345651388168335 (0.2294937939014075) + | > loss_disc_real_2: 0.21490469574928284 (0.2364222820075053) + | > loss_disc_real_3: 0.2403278946876526 (0.23688978654033732) + | > loss_disc_real_4: 0.2509794235229492 (0.2353918637307185) + | > loss_disc_real_5: 0.2510847747325897 (0.23804788325192794) + | > loss_0: 2.873696804046631 (2.7437116146087654) + | > grad_norm_0: tensor(37.5856, device='cuda:0') (tensor(11.0482, device='cuda:0')) + | > loss_gen: 1.7774592638015747 (1.8887854742553998) + | > loss_kl: 2.5158908367156982 (2.469984612374936) + | > loss_feat: 2.950077772140503 (2.9455175597712677) + | > loss_mel: 23.155933380126953 (23.69968039314702) + | > loss_duration: 1.395514726638794 (1.4183755222356544) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.794878005981445 (32.42234363196033) + | > grad_norm_1: tensor(170.5639, device='cuda:0') (tensor(139.1949, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.9211 (0.8456863349338749) + | > loader_time: 0.0149 (0.011862440829007127) + + + --> TIME: 2024-06-25 04:31:20 -- STEP: 290/591 -- GLOBAL_STEP: 6200 + | > loss_disc: 3.003065347671509 (2.7461306859707024) + | > loss_disc_real_0: 0.13686884939670563 (0.21980535179890434) + | > loss_disc_real_1: 0.2789475917816162 (0.22977778896175582) + | > loss_disc_real_2: 0.20593413710594177 (0.2365728781654917) + | > loss_disc_real_3: 0.2202741652727127 (0.23688148953791321) + | > loss_disc_real_4: 0.220094233751297 (0.2356069295056935) + | > loss_disc_real_5: 0.20829953253269196 (0.23806994151452493) + | > loss_0: 3.003065347671509 (2.7461306859707024) + | > grad_norm_0: tensor(18.9528, device='cuda:0') (tensor(10.9507, device='cuda:0')) + | > loss_gen: 2.0900731086730957 (1.8906432521754297) + | > loss_kl: 2.5796971321105957 (2.4671454265199864) + | > loss_feat: 2.7455172538757324 (2.9417182996355256) + | > loss_mel: 23.526596069335938 (23.685243304022432) + | > loss_duration: 1.3611077070236206 (1.4192726459996459) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.3029899597168 (32.40402300604459) + | > grad_norm_1: tensor(93.1679, device='cuda:0') (tensor(143.1475, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.9186 (0.8504848866627134) + | > loader_time: 0.0133 (0.012138910129152495) + + + --> TIME: 2024-06-25 04:31:43 -- STEP: 315/591 -- GLOBAL_STEP: 6225 + | > loss_disc: 3.2353744506835938 (2.7592570706019344) + | > loss_disc_real_0: 0.19551599025726318 (0.22025961750556552) + | > loss_disc_real_1: 0.49254703521728516 (0.23411432322528627) + | > loss_disc_real_2: 0.23891285061836243 (0.2386993591274534) + | > loss_disc_real_3: 0.24688507616519928 (0.23745656883905805) + | > loss_disc_real_4: 0.3080528676509857 (0.23715666830539703) + | > loss_disc_real_5: 0.27129748463630676 (0.23812719522014497) + | > loss_0: 3.2353744506835938 (2.7592570706019344) + | > grad_norm_0: tensor(9.0110, device='cuda:0') (tensor(11.1617, device='cuda:0')) + | > loss_gen: 1.7968348264694214 (1.8962179558617729) + | > loss_kl: 2.3421614170074463 (2.46352078498356) + | > loss_feat: 2.493372917175293 (2.9356841344682) + | > loss_mel: 23.32074737548828 (23.713604954310835) + | > loss_duration: 1.4300098419189453 (1.4198507290037856) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.38312530517578 (32.4288786328028) + | > grad_norm_1: tensor(100.2020, device='cuda:0') (tensor(145.5773, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.9243 (0.8557820607745458) + | > loader_time: 0.0164 (0.012391437046111573) + + + --> TIME: 2024-06-25 04:32:07 -- STEP: 340/591 -- GLOBAL_STEP: 6250 + | > loss_disc: 2.776460647583008 (2.76624623116325) + | > loss_disc_real_0: 0.3133302330970764 (0.2200410693007357) + | > loss_disc_real_1: 0.23475216329097748 (0.23772230698343585) + | > loss_disc_real_2: 0.24538011848926544 (0.23914818001144073) + | > loss_disc_real_3: 0.22896742820739746 (0.23748761827454848) + | > loss_disc_real_4: 0.236606165766716 (0.238374494410613) + | > loss_disc_real_5: 0.2121412456035614 (0.23804370905546582) + | > loss_0: 2.776460647583008 (2.76624623116325) + | > grad_norm_0: tensor(20.0374, device='cuda:0') (tensor(10.9467, device='cuda:0')) + | > loss_gen: 1.6350781917572021 (1.8894259715781492) + | > loss_kl: 2.443361282348633 (2.4603714641402763) + | > loss_feat: 2.5465946197509766 (2.9054283864357884) + | > loss_mel: 23.714221954345703 (23.717966618257407) + | > loss_duration: 1.4453399181365967 (1.4209396986400382) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.784595489501953 (32.394132210226616) + | > grad_norm_1: tensor(92.1923, device='cuda:0') (tensor(143.7805, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.938 (0.8610571244183709) + | > loader_time: 0.0143 (0.012634854456957646) + + + --> TIME: 2024-06-25 04:32:31 -- STEP: 365/591 -- GLOBAL_STEP: 6275 + | > loss_disc: 2.7720582485198975 (2.766842100718251) + | > loss_disc_real_0: 0.19076013565063477 (0.22001308772253664) + | > loss_disc_real_1: 0.20395217835903168 (0.23728470269539584) + | > loss_disc_real_2: 0.25658079981803894 (0.23888715113679024) + | > loss_disc_real_3: 0.2469918131828308 (0.2375327567939889) + | > loss_disc_real_4: 0.22694528102874756 (0.2387910204390957) + | > loss_disc_real_5: 0.2450948804616928 (0.23797042259614762) + | > loss_0: 2.7720582485198975 (2.766842100718251) + | > grad_norm_0: tensor(6.1788, device='cuda:0') (tensor(10.9765, device='cuda:0')) + | > loss_gen: 1.805480718612671 (1.8838402026320158) + | > loss_kl: 2.433349370956421 (2.458013668452225) + | > loss_feat: 2.567929983139038 (2.883006142263542) + | > loss_mel: 23.681926727294922 (23.698221483622508) + | > loss_duration: 1.4383145570755005 (1.4225836956337703) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.927001953125 (32.345665265436054) + | > grad_norm_1: tensor(149.9000, device='cuda:0') (tensor(141.5358, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.957 (0.8667634349979766) + | > loader_time: 0.0167 (0.012905966092462407) + + + --> TIME: 2024-06-25 04:32:56 -- STEP: 390/591 -- GLOBAL_STEP: 6300 + | > loss_disc: 2.745312213897705 (2.765899257782179) + | > loss_disc_real_0: 0.3213220238685608 (0.22026808863649003) + | > loss_disc_real_1: 0.2083958089351654 (0.2367361161380242) + | > loss_disc_real_2: 0.26584315299987793 (0.2386029444443874) + | > loss_disc_real_3: 0.26728835701942444 (0.2372926496160336) + | > loss_disc_real_4: 0.2505885064601898 (0.23889826795993707) + | > loss_disc_real_5: 0.2904040217399597 (0.23800179052811402) + | > loss_0: 2.745312213897705 (2.765899257782179) + | > grad_norm_0: tensor(16.9955, device='cuda:0') (tensor(11.1426, device='cuda:0')) + | > loss_gen: 1.801823616027832 (1.8817495064857679) + | > loss_kl: 2.3740217685699463 (2.4552133609087052) + | > loss_feat: 2.873027801513672 (2.873183678358028) + | > loss_mel: 24.081687927246094 (23.67022845928485) + | > loss_duration: 1.4617104530334473 (1.4235192048243985) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.5922737121582 (32.303894311953854) + | > grad_norm_1: tensor(57.7902, device='cuda:0') (tensor(137.8552, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.9529 (0.8729907005261153) + | > loader_time: 0.0179 (0.013150295844444859) + + + --> TIME: 2024-06-25 04:33:21 -- STEP: 415/591 -- GLOBAL_STEP: 6325 + | > loss_disc: 2.7392940521240234 (2.7639508345040937) + | > loss_disc_real_0: 0.15821772813796997 (0.21991688855082156) + | > loss_disc_real_1: 0.251587837934494 (0.23607621862586722) + | > loss_disc_real_2: 0.26725080609321594 (0.23838739025305553) + | > loss_disc_real_3: 0.22967000305652618 (0.23726155901529702) + | > loss_disc_real_4: 0.2657375931739807 (0.2388336319521249) + | > loss_disc_real_5: 0.26105573773384094 (0.23803211464939347) + | > loss_0: 2.7392940521240234 (2.7639508345040937) + | > grad_norm_0: tensor(15.6467, device='cuda:0') (tensor(10.8870, device='cuda:0')) + | > loss_gen: 1.925994634628296 (1.8811640239623655) + | > loss_kl: 2.367616891860962 (2.4531673787588093) + | > loss_feat: 2.7418463230133057 (2.869646706638565) + | > loss_mel: 23.55190658569336 (23.646234902990866) + | > loss_duration: 1.4431742429733276 (1.4243860465934475) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.03053665161133 (32.27459912587365) + | > grad_norm_1: tensor(98.3710, device='cuda:0') (tensor(135.9057, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 1.0164 (0.8792401170156088) + | > loader_time: 0.0196 (0.013421599261731984) + + + --> TIME: 2024-06-25 04:33:46 -- STEP: 440/591 -- GLOBAL_STEP: 6350 + | > loss_disc: 2.745800733566284 (2.764143751426177) + | > loss_disc_real_0: 0.17782387137413025 (0.22088170292025264) + | > loss_disc_real_1: 0.22306035459041595 (0.23534791599959135) + | > loss_disc_real_2: 0.27130985260009766 (0.23826797421682966) + | > loss_disc_real_3: 0.22056183218955994 (0.23715135187587955) + | > loss_disc_real_4: 0.23965516686439514 (0.23851555782285602) + | > loss_disc_real_5: 0.23669272661209106 (0.23809825412251734) + | > loss_0: 2.745800733566284 (2.764143751426177) + | > grad_norm_0: tensor(5.8574, device='cuda:0') (tensor(11.0434, device='cuda:0')) + | > loss_gen: 1.9206438064575195 (1.880295066941868) + | > loss_kl: 2.438493490219116 (2.4497446358203887) + | > loss_feat: 2.7201449871063232 (2.8647880136966695) + | > loss_mel: 22.97920036315918 (23.625906528126105) + | > loss_duration: 1.425752878189087 (1.4251245103099126) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.484235763549805 (32.24585880799731) + | > grad_norm_1: tensor(79.8653, device='cuda:0') (tensor(135.0362, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 1.0065 (0.886011704531583) + | > loader_time: 0.0205 (0.013717855648560956) + + + --> TIME: 2024-06-25 04:34:12 -- STEP: 465/591 -- GLOBAL_STEP: 6375 + | > loss_disc: 2.730682849884033 (2.763097691279586) + | > loss_disc_real_0: 0.306387722492218 (0.2212073746227449) + | > loss_disc_real_1: 0.2187078893184662 (0.2348414940058544) + | > loss_disc_real_2: 0.2510489225387573 (0.2381258463026375) + | > loss_disc_real_3: 0.22094349563121796 (0.2370523988880137) + | > loss_disc_real_4: 0.23015061020851135 (0.23840653281058036) + | > loss_disc_real_5: 0.2006145864725113 (0.2379765155174399) + | > loss_0: 2.730682849884033 (2.763097691279586) + | > grad_norm_0: tensor(15.6837, device='cuda:0') (tensor(10.8157, device='cuda:0')) + | > loss_gen: 1.7497462034225464 (1.878073900233033) + | > loss_kl: 2.438819646835327 (2.448197067937543) + | > loss_feat: 2.8031249046325684 (2.857510466216712) + | > loss_mel: 22.748069763183594 (23.60132764180501) + | > loss_duration: 1.4460041522979736 (1.4263026055469306) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.18576431274414 (32.21141173249937) + | > grad_norm_1: tensor(68.0406, device='cuda:0') (tensor(134.0410, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 1.0131 (0.8928342814086586) + | > loader_time: 0.017 (0.014025087766749883) + + + --> TIME: 2024-06-25 04:34:39 -- STEP: 490/591 -- GLOBAL_STEP: 6400 + | > loss_disc: 2.7114930152893066 (2.7627423048019413) + | > loss_disc_real_0: 0.18606078624725342 (0.2210517982439119) + | > loss_disc_real_1: 0.22958554327487946 (0.23451936669191537) + | > loss_disc_real_2: 0.21312209963798523 (0.23795280191971332) + | > loss_disc_real_3: 0.22250743210315704 (0.23721723872788097) + | > loss_disc_real_4: 0.23289717733860016 (0.23815402613610637) + | > loss_disc_real_5: 0.25902748107910156 (0.23803807202042365) + | > loss_0: 2.7114930152893066 (2.7627423048019413) + | > grad_norm_0: tensor(6.9209, device='cuda:0') (tensor(10.7331, device='cuda:0')) + | > loss_gen: 1.8560168743133545 (1.8772587284749869) + | > loss_kl: 2.3425681591033936 (2.4455573617195596) + | > loss_feat: 2.869920015335083 (2.854278679283297) + | > loss_mel: 23.553791046142578 (23.589435818730564) + | > loss_duration: 1.4246599674224854 (1.427560753968297) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.04695510864258 (32.19409137647985) + | > grad_norm_1: tensor(78.8074, device='cuda:0') (tensor(134.7188, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 1.0301 (0.9009455758698133) + | > loader_time: 0.0206 (0.014331375336160463) + + + --> TIME: 2024-06-25 04:35:07 -- STEP: 515/591 -- GLOBAL_STEP: 6425 + | > loss_disc: 2.7434725761413574 (2.7622239668392448) + | > loss_disc_real_0: 0.15232069790363312 (0.22118228375911714) + | > loss_disc_real_1: 0.1893867552280426 (0.234135447876546) + | > loss_disc_real_2: 0.22353558242321014 (0.23782928010213722) + | > loss_disc_real_3: 0.26743578910827637 (0.2370934121527718) + | > loss_disc_real_4: 0.24982638657093048 (0.23816854389547143) + | > loss_disc_real_5: 0.2552202343940735 (0.2381075922146584) + | > loss_0: 2.7434725761413574 (2.7622239668392448) + | > grad_norm_0: tensor(12.3543, device='cuda:0') (tensor(10.7546, device='cuda:0')) + | > loss_gen: 1.8496549129486084 (1.8759189533955842) + | > loss_kl: 2.357454299926758 (2.4431518077850343) + | > loss_feat: 2.6565518379211426 (2.8474175763361655) + | > loss_mel: 22.39577293395996 (23.558268134107863) + | > loss_duration: 1.461694598197937 (1.4291192114931863) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.721128463745117 (32.15387572501474) + | > grad_norm_1: tensor(172.8206, device='cuda:0') (tensor(134.0607, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 1.0972 (0.9112134637184514) + | > loader_time: 0.0225 (0.014620883256486318) + + + --> TIME: 2024-06-25 04:35:37 -- STEP: 540/591 -- GLOBAL_STEP: 6450 + | > loss_disc: 2.7289459705352783 (2.760873589692293) + | > loss_disc_real_0: 0.1086108535528183 (0.22084783873072378) + | > loss_disc_real_1: 0.2489064633846283 (0.23385357126869535) + | > loss_disc_real_2: 0.24695566296577454 (0.23779503890761622) + | > loss_disc_real_3: 0.286932110786438 (0.23712942509187593) + | > loss_disc_real_4: 0.21501728892326355 (0.23799842640205665) + | > loss_disc_real_5: 0.21408537030220032 (0.23798110543025863) + | > loss_0: 2.7289459705352783 (2.760873589692293) + | > grad_norm_0: tensor(21.3375, device='cuda:0') (tensor(10.7935, device='cuda:0')) + | > loss_gen: 1.8590848445892334 (1.8756690996664542) + | > loss_kl: 2.278592586517334 (2.44080434596097) + | > loss_feat: 2.82228422164917 (2.844843340361559) + | > loss_mel: 22.518653869628906 (23.53740224485044) + | > loss_duration: 1.4558134078979492 (1.4305582808123691) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.934429168701172 (32.129277345869326) + | > grad_norm_1: tensor(40.7222, device='cuda:0') (tensor(132.8790, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 1.1916 (0.9227463859098929) + | > loader_time: 0.0245 (0.014965589841206868) + + + --> TIME: 2024-06-25 04:36:08 -- STEP: 565/591 -- GLOBAL_STEP: 6475 + | > loss_disc: 2.767651081085205 (2.760038168662417) + | > loss_disc_real_0: 0.21183428168296814 (0.2208650959540257) + | > loss_disc_real_1: 0.24432867765426636 (0.2334755069782248) + | > loss_disc_real_2: 0.24642466008663177 (0.237768926900045) + | > loss_disc_real_3: 0.24627159535884857 (0.2369823494052465) + | > loss_disc_real_4: 0.2587287425994873 (0.2379439048535001) + | > loss_disc_real_5: 0.22155915200710297 (0.2380166621334785) + | > loss_0: 2.767651081085205 (2.760038168662417) + | > grad_norm_0: tensor(4.0157, device='cuda:0') (tensor(10.8402, device='cuda:0')) + | > loss_gen: 1.8224116563796997 (1.8750005747364686) + | > loss_kl: 2.4084389209747314 (2.4380094739188136) + | > loss_feat: 2.555622100830078 (2.8403778291381556) + | > loss_mel: 22.241769790649414 (23.5103685092082) + | > loss_duration: 1.488750696182251 (1.4320078161965426) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.516992568969727 (32.09576425467977) + | > grad_norm_1: tensor(205.2759, device='cuda:0') (tensor(132.4428, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 1.3405 (0.9362486873052817) + | > loader_time: 0.022 (0.015328670181004348) + + + --> TIME: 2024-06-25 04:36:40 -- STEP: 590/591 -- GLOBAL_STEP: 6500 + | > loss_disc: 2.8231050968170166 (2.7591472520666604) + | > loss_disc_real_0: 0.1895408034324646 (0.22072613643387606) + | > loss_disc_real_1: 0.21311721205711365 (0.2332516174826581) + | > loss_disc_real_2: 0.2768353223800659 (0.2377021004840479) + | > loss_disc_real_3: 0.2568807005882263 (0.2369547044826766) + | > loss_disc_real_4: 0.2820327877998352 (0.23785038974325537) + | > loss_disc_real_5: 0.25302407145500183 (0.2379343686214948) + | > loss_0: 2.8231050968170166 (2.7591472520666604) + | > grad_norm_0: tensor(8.1389, device='cuda:0') (tensor(10.8291, device='cuda:0')) + | > loss_gen: 1.83424973487854 (1.874469062231355) + | > loss_kl: 2.376258373260498 (2.4354186300504006) + | > loss_feat: 2.799631357192993 (2.835866904662826) + | > loss_mel: 23.05574607849121 (23.49463165735793) + | > loss_duration: 1.4461243152618408 (1.433543948601868) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.51201057434082 (32.073930252204555) + | > grad_norm_1: tensor(186.9107, device='cuda:0') (tensor(133.2832, device='cuda:0')) + | > current_lr_0: 0.00019975014057813518 + | > current_lr_1: 0.00019975014057813518 + | > step_time: 0.4683 (0.9495556891974756) + | > loader_time: 0.0078 (0.015692939596661067) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003557524782546023 (-2.604849795077727e-05) + | > avg_loss_disc: 2.7644374319847596 (+0.008102706138124027) + | > avg_loss_disc_real_0: 0.24895745452414167 (-0.004611234398598424) + | > avg_loss_disc_real_1: 0.19298947554953555 (-0.005183606705767035) + | > avg_loss_disc_real_2: 0.21878057971913764 (-0.03946968722850727) + | > avg_loss_disc_real_3: 0.21385466640299938 (+0.0029689273301591046) + | > avg_loss_disc_real_4: 0.21561736883001126 (-0.009122610726255032) + | > avg_loss_disc_real_5: 0.2508093565702439 (+0.02318088773717278) + | > avg_loss_0: 2.7644374319847596 (+0.008102706138124027) + | > avg_loss_gen: 1.766080582395513 (-0.04398861590852121) + | > avg_loss_kl: 2.5360006220797273 (+0.09255083571089129) + | > avg_loss_feat: 2.8111944705881973 (+0.015387565531628677) + | > avg_loss_mel: 23.246235705436543 (-0.44785495514565454) + | > avg_loss_duration: 1.4141955629308167 (-0.011179665301709107) + | > avg_loss_1: 31.773706882557974 (-0.39508531448687734) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_6501.pth + + > EPOCH: 12/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 04:36:54)  + + --> TIME: 2024-06-25 04:37:16 -- STEP: 24/591 -- GLOBAL_STEP: 6525 + | > loss_disc: 2.891573429107666 (2.738965481519699) + | > loss_disc_real_0: 0.3646014928817749 (0.22180199809372425) + | > loss_disc_real_1: 0.19836029410362244 (0.22743034673233828) + | > loss_disc_real_2: 0.25355786085128784 (0.2361818440258503) + | > loss_disc_real_3: 0.2283211499452591 (0.24062628795703253) + | > loss_disc_real_4: 0.19726839661598206 (0.23398689553141594) + | > loss_disc_real_5: 0.19928044080734253 (0.23861773498356342) + | > loss_0: 2.891573429107666 (2.738965481519699) + | > grad_norm_0: tensor(24.1674, device='cuda:0') (tensor(10.3523, device='cuda:0')) + | > loss_gen: 1.6826550960540771 (1.8872619221607845) + | > loss_kl: 2.6023714542388916 (2.6198914448420205) + | > loss_feat: 2.9373433589935303 (3.0670083463191986) + | > loss_mel: 23.347618103027344 (23.468663851420086) + | > loss_duration: 1.3841670751571655 (1.3847801834344864) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.95415496826172 (32.42760554949441) + | > grad_norm_1: tensor(129.3966, device='cuda:0') (tensor(127.7846, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.7837 (0.7780842383702596) + | > loader_time: 0.0095 (0.009033987919489544) + + + --> TIME: 2024-06-25 04:37:36 -- STEP: 49/591 -- GLOBAL_STEP: 6550 + | > loss_disc: 2.788651943206787 (2.737415294257962) + | > loss_disc_real_0: 0.35132256150245667 (0.2182662677095861) + | > loss_disc_real_1: 0.24442894756793976 (0.22864184726257714) + | > loss_disc_real_2: 0.24964837729930878 (0.23515238719327108) + | > loss_disc_real_3: 0.2531715929508209 (0.23822867353351748) + | > loss_disc_real_4: 0.23988620936870575 (0.23611017939995746) + | > loss_disc_real_5: 0.21963448822498322 (0.2389669488279187) + | > loss_0: 2.788651943206787 (2.737415294257962) + | > grad_norm_0: tensor(22.2901, device='cuda:0') (tensor(9.9222, device='cuda:0')) + | > loss_gen: 1.7488352060317993 (1.8811671295944525) + | > loss_kl: 2.3522984981536865 (2.576476676123482) + | > loss_feat: 3.0795891284942627 (3.007422228248752) + | > loss_mel: 23.125144958496094 (23.34442594100018) + | > loss_duration: 1.4200866222381592 (1.3988476894339736) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.725954055786133 (32.208339652236596) + | > grad_norm_1: tensor(174.3826, device='cuda:0') (tensor(134.0723, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.7972 (0.7888055334285814) + | > loader_time: 0.0096 (0.00952102700058295) + + + --> TIME: 2024-06-25 04:37:57 -- STEP: 74/591 -- GLOBAL_STEP: 6575 + | > loss_disc: 2.7282907962799072 (2.743901745693104) + | > loss_disc_real_0: 0.19205011427402496 (0.21985663262170715) + | > loss_disc_real_1: 0.2736559510231018 (0.22881567075445847) + | > loss_disc_real_2: 0.25323259830474854 (0.23596710530487267) + | > loss_disc_real_3: 0.26011186838150024 (0.2380952841124019) + | > loss_disc_real_4: 0.2117333561182022 (0.23559971153736115) + | > loss_disc_real_5: 0.23197482526302338 (0.2382331146179019) + | > loss_0: 2.7282907962799072 (2.743901745693104) + | > grad_norm_0: tensor(6.4295, device='cuda:0') (tensor(10.7706, device='cuda:0')) + | > loss_gen: 1.8805066347122192 (1.8804646150485889) + | > loss_kl: 2.6056911945343018 (2.5555349298425614) + | > loss_feat: 3.0728039741516113 (2.9917190010483203) + | > loss_mel: 23.805011749267578 (23.390375034229177) + | > loss_duration: 1.4257700443267822 (1.404615273346772) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.7897834777832 (32.22270888251225) + | > grad_norm_1: tensor(217.1349, device='cuda:0') (tensor(137.6812, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.8086 (0.7974294166307191) + | > loader_time: 0.0108 (0.009900673015697583) + + + --> TIME: 2024-06-25 04:38:18 -- STEP: 99/591 -- GLOBAL_STEP: 6600 + | > loss_disc: 2.7622480392456055 (2.7439155819440133) + | > loss_disc_real_0: 0.21899637579917908 (0.21995626788849781) + | > loss_disc_real_1: 0.2885146141052246 (0.22844301841475748) + | > loss_disc_real_2: 0.2494022250175476 (0.23608023214219798) + | > loss_disc_real_3: 0.25586479902267456 (0.23719506793551975) + | > loss_disc_real_4: 0.26442134380340576 (0.23586460240561552) + | > loss_disc_real_5: 0.2561805844306946 (0.23862451420287895) + | > loss_0: 2.7622480392456055 (2.7439155819440133) + | > grad_norm_0: tensor(5.0440, device='cuda:0') (tensor(10.3315, device='cuda:0')) + | > loss_gen: 1.9087175130844116 (1.8784714612093838) + | > loss_kl: 2.3921563625335693 (2.537641050839665) + | > loss_feat: 2.815045118331909 (2.9868231758926855) + | > loss_mel: 23.47801971435547 (23.374607471504596) + | > loss_duration: 1.4190874099731445 (1.4131785318104908) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.01302719116211 (32.19072174303457) + | > grad_norm_1: tensor(95.8657, device='cuda:0') (tensor(132.0870, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.8282 (0.8066101700368551) + | > loader_time: 0.0106 (0.010311911804507475) + + + --> TIME: 2024-06-25 04:38:40 -- STEP: 124/591 -- GLOBAL_STEP: 6625 + | > loss_disc: 2.6960372924804688 (2.7435403985361906) + | > loss_disc_real_0: 0.1990731656551361 (0.2203621452734355) + | > loss_disc_real_1: 0.21612168848514557 (0.22809272403678588) + | > loss_disc_real_2: 0.24503080546855927 (0.2359410322481586) + | > loss_disc_real_3: 0.2524637281894684 (0.23673635408762964) + | > loss_disc_real_4: 0.22762885689735413 (0.23507809554857592) + | > loss_disc_real_5: 0.21396324038505554 (0.23819812543449864) + | > loss_0: 2.6960372924804688 (2.7435403985361906) + | > grad_norm_0: tensor(3.9793, device='cuda:0') (tensor(10.5649, device='cuda:0')) + | > loss_gen: 1.8938539028167725 (1.8767395702100569) + | > loss_kl: 2.5835647583007812 (2.517923426243567) + | > loss_feat: 2.9481754302978516 (2.982026396259185) + | > loss_mel: 22.64286994934082 (23.390711230616414) + | > loss_duration: 1.4320440292358398 (1.4150360457358822) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.500507354736328 (32.1824367123265) + | > grad_norm_1: tensor(96.0841, device='cuda:0') (tensor(129.9857, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.8411 (0.8140248925455152) + | > loader_time: 0.0125 (0.010605052594215637) + + + --> TIME: 2024-06-25 04:39:01 -- STEP: 149/591 -- GLOBAL_STEP: 6650 + | > loss_disc: 2.68904185295105 (2.7451557265032056) + | > loss_disc_real_0: 0.24634480476379395 (0.22018844534286716) + | > loss_disc_real_1: 0.2178766280412674 (0.22876964649498052) + | > loss_disc_real_2: 0.2091483622789383 (0.23714898546670107) + | > loss_disc_real_3: 0.1863674819469452 (0.23652068280533656) + | > loss_disc_real_4: 0.25468710064888 (0.2351947256782711) + | > loss_disc_real_5: 0.23586830496788025 (0.23852892400034323) + | > loss_0: 2.68904185295105 (2.7451557265032056) + | > grad_norm_0: tensor(7.0061, device='cuda:0') (tensor(10.6331, device='cuda:0')) + | > loss_gen: 1.815079927444458 (1.8794452280006153) + | > loss_kl: 2.473435640335083 (2.508139616691026) + | > loss_feat: 3.003598690032959 (2.9800400749949) + | > loss_mel: 24.005617141723633 (23.367513106173316) + | > loss_duration: 1.4313184022903442 (1.4146511986751684) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.72904968261719 (32.14978928533973) + | > grad_norm_1: tensor(76.7908, device='cuda:0') (tensor(126.4517, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.8504 (0.8201063319340647) + | > loader_time: 0.0113 (0.010738350400988686) + + + --> TIME: 2024-06-25 04:39:23 -- STEP: 174/591 -- GLOBAL_STEP: 6675 + | > loss_disc: 2.7793338298797607 (2.747228083939387) + | > loss_disc_real_0: 0.16453798115253448 (0.22028181427854232) + | > loss_disc_real_1: 0.27647706866264343 (0.22893467237209444) + | > loss_disc_real_2: 0.25105005502700806 (0.23727477770084623) + | > loss_disc_real_3: 0.26685193181037903 (0.23645795684778828) + | > loss_disc_real_4: 0.23960357904434204 (0.23545218730109863) + | > loss_disc_real_5: 0.24033229053020477 (0.23857249602161604) + | > loss_0: 2.7793338298797607 (2.747228083939387) + | > grad_norm_0: tensor(11.7759, device='cuda:0') (tensor(10.8964, device='cuda:0')) + | > loss_gen: 1.729177474975586 (1.8776853502481834) + | > loss_kl: 2.4550273418426514 (2.496042428345515) + | > loss_feat: 2.8165884017944336 (2.970921220450566) + | > loss_mel: 22.798782348632812 (23.351586911870147) + | > loss_duration: 1.4261500835418701 (1.4127243380436951) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.225725173950195 (32.108960381869586) + | > grad_norm_1: tensor(214.7711, device='cuda:0') (tensor(129.9280, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.8735 (0.8258771581211307) + | > loader_time: 0.0123 (0.010852427318178376) + + + --> TIME: 2024-06-25 04:39:45 -- STEP: 199/591 -- GLOBAL_STEP: 6700 + | > loss_disc: 2.7337992191314697 (2.7484756582346384) + | > loss_disc_real_0: 0.17442849278450012 (0.21978668119739647) + | > loss_disc_real_1: 0.20735539495944977 (0.22919602714591283) + | > loss_disc_real_2: 0.2115996778011322 (0.2379563747338913) + | > loss_disc_real_3: 0.23602132499217987 (0.23647039926531327) + | > loss_disc_real_4: 0.24687980115413666 (0.23559709454900654) + | > loss_disc_real_5: 0.22864344716072083 (0.2387645654941923) + | > loss_0: 2.7337992191314697 (2.7484756582346384) + | > grad_norm_0: tensor(8.1852, device='cuda:0') (tensor(10.6913, device='cuda:0')) + | > loss_gen: 1.8394854068756104 (1.8795849767761614) + | > loss_kl: 2.4645609855651855 (2.4922664165496826) + | > loss_feat: 2.9134321212768555 (2.963046025990242) + | > loss_mel: 22.82463836669922 (23.35100169157862) + | > loss_duration: 1.4047776460647583 (1.4122170245827144) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.4468936920166 (32.09811626606846) + | > grad_norm_1: tensor(154.3432, device='cuda:0') (tensor(134.0220, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.8677 (0.8318403486031383) + | > loader_time: 0.012 (0.011000865667908635) + + + --> TIME: 2024-06-25 04:40:08 -- STEP: 224/591 -- GLOBAL_STEP: 6725 + | > loss_disc: 2.7650861740112305 (2.750057423753397) + | > loss_disc_real_0: 0.2638319432735443 (0.21956129458599857) + | > loss_disc_real_1: 0.2598686218261719 (0.23005841412980635) + | > loss_disc_real_2: 0.21781578660011292 (0.2384922237002424) + | > loss_disc_real_3: 0.22471873462200165 (0.23637683616418922) + | > loss_disc_real_4: 0.2896650433540344 (0.23550628045839922) + | > loss_disc_real_5: 0.2078706920146942 (0.23852535217468226) + | > loss_0: 2.7650861740112305 (2.750057423753397) + | > grad_norm_0: tensor(8.3512, device='cuda:0') (tensor(10.6178, device='cuda:0')) + | > loss_gen: 1.8967972993850708 (1.8790823556482792) + | > loss_kl: 2.386500358581543 (2.488153286278248) + | > loss_feat: 2.8953938484191895 (2.9549264907836914) + | > loss_mel: 23.387304306030273 (23.364963812487467) + | > loss_duration: 1.4134089946746826 (1.4112717950982707) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.97940444946289 (32.09839786802014) + | > grad_norm_1: tensor(104.8873, device='cuda:0') (tensor(135.6474, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.8931 (0.8370026999286242) + | > loader_time: 0.0148 (0.01128254724400384) + + + --> TIME: 2024-06-25 04:40:31 -- STEP: 249/591 -- GLOBAL_STEP: 6750 + | > loss_disc: 2.716949462890625 (2.747995168808473) + | > loss_disc_real_0: 0.15942272543907166 (0.21902883989384855) + | > loss_disc_real_1: 0.25107038021087646 (0.23000371318504986) + | > loss_disc_real_2: 0.24299928545951843 (0.23811729360057646) + | > loss_disc_real_3: 0.22682081162929535 (0.236363164093121) + | > loss_disc_real_4: 0.22274471819400787 (0.2354556140171955) + | > loss_disc_real_5: 0.30020514130592346 (0.238351331118599) + | > loss_0: 2.716949462890625 (2.747995168808473) + | > grad_norm_0: tensor(11.2669, device='cuda:0') (tensor(10.6106, device='cuda:0')) + | > loss_gen: 1.916882038116455 (1.8789150561673573) + | > loss_kl: 2.43800950050354 (2.484651664174704) + | > loss_feat: 2.809016466140747 (2.954545606092277) + | > loss_mel: 23.056791305541992 (23.356047779680736) + | > loss_duration: 1.4334852695465088 (1.411813104966558) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.654184341430664 (32.08597326470183) + | > grad_norm_1: tensor(179.0044, device='cuda:0') (tensor(132.6968, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.902 (0.8423679947374335) + | > loader_time: 0.0149 (0.011574025134963683) + + + --> TIME: 2024-06-25 04:40:54 -- STEP: 274/591 -- GLOBAL_STEP: 6775 + | > loss_disc: 2.793846607208252 (2.7477065063741075) + | > loss_disc_real_0: 0.18490999937057495 (0.21919070904816157) + | > loss_disc_real_1: 0.2088608741760254 (0.22930887629733457) + | > loss_disc_real_2: 0.2734958827495575 (0.23799494469035282) + | > loss_disc_real_3: 0.23390525579452515 (0.2362940124895451) + | > loss_disc_real_4: 0.22377435863018036 (0.23547831150519588) + | > loss_disc_real_5: 0.20530983805656433 (0.23822184320348894) + | > loss_0: 2.793846607208252 (2.7477065063741075) + | > grad_norm_0: tensor(6.5701, device='cuda:0') (tensor(10.8635, device='cuda:0')) + | > loss_gen: 1.9403208494186401 (1.878261622286191) + | > loss_kl: 2.394040584564209 (2.4797076709079047) + | > loss_feat: 2.8256168365478516 (2.950194029912462) + | > loss_mel: 23.465742111206055 (23.334095189171116) + | > loss_duration: 1.4371967315673828 (1.4116573338090939) + | > amp_scaler: 1024.0 (521.3430656934306) + | > loss_1: 32.06291961669922 (32.0539158800222) + | > grad_norm_1: tensor(48.3576, device='cuda:0') (tensor(131.1890, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.8964 (0.8475920231672968) + | > loader_time: 0.0182 (0.01185561792693869) + + + --> TIME: 2024-06-25 04:41:17 -- STEP: 299/591 -- GLOBAL_STEP: 6800 + | > loss_disc: 2.757688283920288 (2.7462311548532843) + | > loss_disc_real_0: 0.1507899910211563 (0.21863263897772217) + | > loss_disc_real_1: 0.25690144300460815 (0.22953567158019653) + | > loss_disc_real_2: 0.24058762192726135 (0.23752462046600903) + | > loss_disc_real_3: 0.2575571537017822 (0.2364203888038329) + | > loss_disc_real_4: 0.21865665912628174 (0.2354804947224748) + | > loss_disc_real_5: 0.257051020860672 (0.23832267717174863) + | > loss_0: 2.757688283920288 (2.7462311548532843) + | > grad_norm_0: tensor(14.8651, device='cuda:0') (tensor(10.6573, device='cuda:0')) + | > loss_gen: 1.8880095481872559 (1.8775185741309737) + | > loss_kl: 2.4362123012542725 (2.4754158980072942) + | > loss_feat: 2.9222798347473145 (2.947840547880601) + | > loss_mel: 22.854686737060547 (23.312576121709817) + | > loss_duration: 1.4140841960906982 (1.4119013699400786) + | > amp_scaler: 1024.0 (563.3712374581941) + | > loss_1: 31.51527214050293 (32.02525254316548) + | > grad_norm_1: tensor(182.7087, device='cuda:0') (tensor(129.6714, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.9255 (0.8532701639028692) + | > loader_time: 0.0165 (0.01215264629759517) + + + --> TIME: 2024-06-25 04:41:41 -- STEP: 324/591 -- GLOBAL_STEP: 6825 + | > loss_disc: 2.788172721862793 (2.7470564062212715) + | > loss_disc_real_0: 0.2901095151901245 (0.21943905712737713) + | > loss_disc_real_1: 0.2146550863981247 (0.2292050843437512) + | > loss_disc_real_2: 0.22404184937477112 (0.2375077919367655) + | > loss_disc_real_3: 0.22377344965934753 (0.23626980521244767) + | > loss_disc_real_4: 0.21971748769283295 (0.23534457449927743) + | > loss_disc_real_5: 0.21119044721126556 (0.23823826162167538) + | > loss_0: 2.788172721862793 (2.7470564062212715) + | > grad_norm_0: tensor(10.2660, device='cuda:0') (tensor(11.0150, device='cuda:0')) + | > loss_gen: 1.9356844425201416 (1.8765801890396778) + | > loss_kl: 2.5425820350646973 (2.474766856358375) + | > loss_feat: 2.923269271850586 (2.9426468213399253) + | > loss_mel: 22.691720962524414 (23.312939579104196) + | > loss_duration: 1.4124194383621216 (1.4126794673042535) + | > amp_scaler: 512.0 (565.7283950617282) + | > loss_1: 31.50567626953125 (32.01961297164725) + | > grad_norm_1: tensor(104.5825, device='cuda:0') (tensor(129.0196, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.9421 (0.8588322739542263) + | > loader_time: 0.0172 (0.01242452565534615) + + + --> TIME: 2024-06-25 04:42:05 -- STEP: 349/591 -- GLOBAL_STEP: 6850 + | > loss_disc: 2.95902156829834 (2.7472246226062054) + | > loss_disc_real_0: 0.12302291393280029 (0.21891277518262153) + | > loss_disc_real_1: 0.3110937178134918 (0.22941565287318125) + | > loss_disc_real_2: 0.21703045070171356 (0.23768675762807742) + | > loss_disc_real_3: 0.25199317932128906 (0.23640343320574664) + | > loss_disc_real_4: 0.2006964236497879 (0.23542259893991202) + | > loss_disc_real_5: 0.19467265903949738 (0.23813607819100846) + | > loss_0: 2.95902156829834 (2.7472246226062054) + | > grad_norm_0: tensor(21.4147, device='cuda:0') (tensor(10.9259, device='cuda:0')) + | > loss_gen: 2.0043861865997314 (1.877408681079788) + | > loss_kl: 2.420419454574585 (2.4716606194788526) + | > loss_feat: 2.799503803253174 (2.936063036194504) + | > loss_mel: 23.759597778320312 (23.305605016670103) + | > loss_duration: 1.4289735555648804 (1.4140375039638977) + | > amp_scaler: 512.0 (561.8796561604582) + | > loss_1: 32.412879943847656 (32.00477485875342) + | > grad_norm_1: tensor(127.8355, device='cuda:0') (tensor(133.4785, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.9489 (0.8644579297150442) + | > loader_time: 0.0166 (0.0126522905847063) + + + --> TIME: 2024-06-25 04:42:29 -- STEP: 374/591 -- GLOBAL_STEP: 6875 + | > loss_disc: 2.8229739665985107 (2.7522637015358) + | > loss_disc_real_0: 0.21469219028949738 (0.21894293742622922) + | > loss_disc_real_1: 0.30861911177635193 (0.23183779396554996) + | > loss_disc_real_2: 0.20829550921916962 (0.23804424037748478) + | > loss_disc_real_3: 0.2503708600997925 (0.2366324195010777) + | > loss_disc_real_4: 0.25779107213020325 (0.23589880939792185) + | > loss_disc_real_5: 0.2593547999858856 (0.23832580714142895) + | > loss_0: 2.8229739665985107 (2.7522637015358) + | > grad_norm_0: tensor(6.0218, device='cuda:0') (tensor(10.9324, device='cuda:0')) + | > loss_gen: 2.0468251705169678 (1.8807843327522278) + | > loss_kl: 2.4600865840911865 (2.4687546096383564) + | > loss_feat: 2.659529447555542 (2.930905987234677) + | > loss_mel: 23.581369400024414 (23.320215215019992) + | > loss_duration: 1.4040367603302002 (1.415440168291491) + | > amp_scaler: 512.0 (558.5454545454545) + | > loss_1: 32.15184783935547 (32.016100322498914) + | > grad_norm_1: tensor(50.0776, device='cuda:0') (tensor(137.0108, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.9511 (0.8701822426229872) + | > loader_time: 0.0172 (0.012931863891887153) + + + --> TIME: 2024-06-25 04:42:54 -- STEP: 399/591 -- GLOBAL_STEP: 6900 + | > loss_disc: 2.7303547859191895 (2.7591023212088666) + | > loss_disc_real_0: 0.19120386242866516 (0.21920991520908542) + | > loss_disc_real_1: 0.26602548360824585 (0.2341242361449657) + | > loss_disc_real_2: 0.2584295868873596 (0.23910448795422576) + | > loss_disc_real_3: 0.22074393928050995 (0.23705790869723586) + | > loss_disc_real_4: 0.24805960059165955 (0.23633175905336412) + | > loss_disc_real_5: 0.24095246195793152 (0.2384023830705418) + | > loss_0: 2.7303547859191895 (2.7591023212088666) + | > grad_norm_0: tensor(5.6014, device='cuda:0') (tensor(10.9737, device='cuda:0')) + | > loss_gen: 1.8649179935455322 (1.8781937946353042) + | > loss_kl: 2.3425962924957275 (2.465697735473326) + | > loss_feat: 2.6540451049804688 (2.9117164677545846) + | > loss_mel: 23.496204376220703 (23.341637436907384) + | > loss_duration: 1.426804542541504 (1.4164269018292737) + | > amp_scaler: 512.0 (555.6290726817042) + | > loss_1: 31.784568786621094 (32.01367234825183) + | > grad_norm_1: tensor(117.2849, device='cuda:0') (tensor(136.8774, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 0.9984 (0.8766723641178061) + | > loader_time: 0.0161 (0.013200942137486352) + + + --> TIME: 2024-06-25 04:43:19 -- STEP: 424/591 -- GLOBAL_STEP: 6925 + | > loss_disc: 2.6859846115112305 (2.758754827503887) + | > loss_disc_real_0: 0.10854272544384003 (0.21898245746445544) + | > loss_disc_real_1: 0.23778249323368073 (0.23383116052609002) + | > loss_disc_real_2: 0.22321370244026184 (0.23906647705666298) + | > loss_disc_real_3: 0.18985404074192047 (0.23692211536866314) + | > loss_disc_real_4: 0.19472207129001617 (0.23630941304255207) + | > loss_disc_real_5: 0.21507446467876434 (0.23829138011864895) + | > loss_0: 2.6859846115112305 (2.758754827503887) + | > grad_norm_0: tensor(21.4612, device='cuda:0') (tensor(10.9885, device='cuda:0')) + | > loss_gen: 2.0164175033569336 (1.8753890426091429) + | > loss_kl: 2.44853138923645 (2.4621761553692347) + | > loss_feat: 3.0927882194519043 (2.8977514623471032) + | > loss_mel: 23.289714813232422 (23.3135798157386) + | > loss_duration: 1.4467273950576782 (1.4170533098139864) + | > amp_scaler: 512.0 (553.0566037735848) + | > loss_1: 32.29418182373047 (31.965949800779153) + | > grad_norm_1: tensor(51.8530, device='cuda:0') (tensor(134.1310, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 1.0071 (0.8834609198120402) + | > loader_time: 0.0165 (0.013481114832860117) + + + --> TIME: 2024-06-25 04:43:45 -- STEP: 449/591 -- GLOBAL_STEP: 6950 + | > loss_disc: 2.784050464630127 (2.75756650671927) + | > loss_disc_real_0: 0.1506805270910263 (0.2189980868938771) + | > loss_disc_real_1: 0.2642185389995575 (0.2333879424528714) + | > loss_disc_real_2: 0.24417968094348907 (0.2388268656746582) + | > loss_disc_real_3: 0.3060806095600128 (0.23709930194114523) + | > loss_disc_real_4: 0.20150314271450043 (0.23617358303282468) + | > loss_disc_real_5: 0.21529364585876465 (0.2381995225404047) + | > loss_0: 2.784050464630127 (2.75756650671927) + | > grad_norm_0: tensor(12.7149, device='cuda:0') (tensor(11.0088, device='cuda:0')) + | > loss_gen: 1.986072301864624 (1.8746843985831552) + | > loss_kl: 2.4911246299743652 (2.4614835725329747) + | > loss_feat: 2.899005174636841 (2.8935810643474356) + | > loss_mel: 22.991193771362305 (23.296974343552613) + | > loss_duration: 1.4277129173278809 (1.4180142385656966) + | > amp_scaler: 512.0 (550.7706013363027) + | > loss_1: 31.79511070251465 (31.944737648910753) + | > grad_norm_1: tensor(94.6342, device='cuda:0') (tensor(133.2952, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 1.056 (0.8903873211025923) + | > loader_time: 0.0153 (0.013824139512195882) + + + --> TIME: 2024-06-25 04:44:11 -- STEP: 474/591 -- GLOBAL_STEP: 6975 + | > loss_disc: 2.7927353382110596 (2.757134252962683) + | > loss_disc_real_0: 0.20409612357616425 (0.2192654406754025) + | > loss_disc_real_1: 0.27799350023269653 (0.23292322687899003) + | > loss_disc_real_2: 0.28360071778297424 (0.23869529372780635) + | > loss_disc_real_3: 0.20817747712135315 (0.23690823069478892) + | > loss_disc_real_4: 0.22133168578147888 (0.23618971939957092) + | > loss_disc_real_5: 0.22194251418113708 (0.23825289261869237) + | > loss_0: 2.7927353382110596 (2.757134252962683) + | > grad_norm_0: tensor(5.2361, device='cuda:0') (tensor(10.9937, device='cuda:0')) + | > loss_gen: 1.9766322374343872 (1.8732001663260318) + | > loss_kl: 2.411102294921875 (2.458084522420342) + | > loss_feat: 2.849486827850342 (2.8876828065904387) + | > loss_mel: 22.930389404296875 (23.27655194680901) + | > loss_duration: 1.4051296710968018 (1.419181276474322) + | > amp_scaler: 512.0 (548.7257383966241) + | > loss_1: 31.572742462158203 (31.914700745530197) + | > grad_norm_1: tensor(60.6392, device='cuda:0') (tensor(132.1036, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 1.013 (0.8973297952096672) + | > loader_time: 0.0212 (0.014118174460366806) + + + --> TIME: 2024-06-25 04:44:38 -- STEP: 499/591 -- GLOBAL_STEP: 7000 + | > loss_disc: 2.704310178756714 (2.7571267379309696) + | > loss_disc_real_0: 0.22157111763954163 (0.21955376375235156) + | > loss_disc_real_1: 0.28171488642692566 (0.23265287132265566) + | > loss_disc_real_2: 0.22986076772212982 (0.23850382306054982) + | > loss_disc_real_3: 0.2697142958641052 (0.2369599150631853) + | > loss_disc_real_4: 0.22985339164733887 (0.23637289203001646) + | > loss_disc_real_5: 0.22215071320533752 (0.23831881585723175) + | > loss_0: 2.704310178756714 (2.7571267379309696) + | > grad_norm_0: tensor(4.6768, device='cuda:0') (tensor(10.9324, device='cuda:0')) + | > loss_gen: 1.8157472610473633 (1.8719382202458048) + | > loss_kl: 2.4180667400360107 (2.4564392432898923) + | > loss_feat: 2.935797691345215 (2.8823910461876823) + | > loss_mel: 23.339967727661133 (23.253153156899728) + | > loss_duration: 1.430964708328247 (1.4204940917735596) + | > amp_scaler: 512.0 (546.885771543086) + | > loss_1: 31.94054412841797 (31.884415796620026) + | > grad_norm_1: tensor(63.9078, device='cuda:0') (tensor(131.4170, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 1.1177 (0.9060139794626788) + | > loader_time: 0.0184 (0.014418149520017817) + + + --> TIME: 2024-06-25 04:45:07 -- STEP: 524/591 -- GLOBAL_STEP: 7025 + | > loss_disc: 2.754969596862793 (2.757384606900105) + | > loss_disc_real_0: 0.239919975399971 (0.21964654566734346) + | > loss_disc_real_1: 0.2577383816242218 (0.23250626481159492) + | > loss_disc_real_2: 0.2115711271762848 (0.23851162502328857) + | > loss_disc_real_3: 0.22339333593845367 (0.2369315920269671) + | > loss_disc_real_4: 0.2516357898712158 (0.23628249352101152) + | > loss_disc_real_5: 0.25171172618865967 (0.23834292007652855) + | > loss_0: 2.754969596862793 (2.757384606900105) + | > grad_norm_0: tensor(6.0818, device='cuda:0') (tensor(11.0086, device='cuda:0')) + | > loss_gen: 1.8972512483596802 (1.8708280036012634) + | > loss_kl: 2.4399006366729736 (2.453420669067905) + | > loss_feat: 2.6877315044403076 (2.876255358903463) + | > loss_mel: 23.212270736694336 (23.229151536490164) + | > loss_duration: 1.4752638339996338 (1.4221445217387376) + | > amp_scaler: 512.0 (545.2213740458013) + | > loss_1: 31.712417602539062 (31.85180012506376) + | > grad_norm_1: tensor(125.2085, device='cuda:0') (tensor(131.1995, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 1.2013 (0.9169499610216559) + | > loader_time: 0.0239 (0.01472125344604026) + + + --> TIME: 2024-06-25 04:45:38 -- STEP: 549/591 -- GLOBAL_STEP: 7050 + | > loss_disc: 2.7186391353607178 (2.756333821892521) + | > loss_disc_real_0: 0.2082454264163971 (0.21949061022593025) + | > loss_disc_real_1: 0.21048173308372498 (0.2323096118365263) + | > loss_disc_real_2: 0.23193860054016113 (0.23838724350668694) + | > loss_disc_real_3: 0.20117302238941193 (0.23684471148067915) + | > loss_disc_real_4: 0.23261860013008118 (0.2361705308349189) + | > loss_disc_real_5: 0.2409597486257553 (0.23829765121142069) + | > loss_0: 2.7186391353607178 (2.756333821892521) + | > grad_norm_0: tensor(4.6434, device='cuda:0') (tensor(10.9447, device='cuda:0')) + | > loss_gen: 1.864136815071106 (1.869731379988415) + | > loss_kl: 2.405634880065918 (2.451842736676741) + | > loss_feat: 2.7055587768554688 (2.869979693807106) + | > loss_mel: 21.882701873779297 (23.207220292916492) + | > loss_duration: 1.432766318321228 (1.423504335198464) + | > amp_scaler: 512.0 (543.7085610200363) + | > loss_1: 30.290800094604492 (31.822278476152196) + | > grad_norm_1: tensor(73.8747, device='cuda:0') (tensor(131.1050, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 1.2394 (0.9301985065793728) + | > loader_time: 0.0233 (0.015108649630798452) + + + --> TIME: 2024-06-25 04:46:11 -- STEP: 574/591 -- GLOBAL_STEP: 7075 + | > loss_disc: 2.7844345569610596 (2.7572813391270117) + | > loss_disc_real_0: 0.1213105320930481 (0.2197792185697614) + | > loss_disc_real_1: 0.23597124218940735 (0.23219342640104604) + | > loss_disc_real_2: 0.23361651599407196 (0.23867069421018042) + | > loss_disc_real_3: 0.22826826572418213 (0.23698471761746684) + | > loss_disc_real_4: 0.2448340803384781 (0.2360949255915469) + | > loss_disc_real_5: 0.26464909315109253 (0.23820863591461647) + | > loss_0: 2.7844345569610596 (2.7572813391270117) + | > grad_norm_0: tensor(16.9808, device='cuda:0') (tensor(11.0815, device='cuda:0')) + | > loss_gen: 1.6528494358062744 (1.8690674331960777) + | > loss_kl: 2.3412718772888184 (2.44941617719803) + | > loss_feat: 2.518678665161133 (2.8636569735895887) + | > loss_mel: 22.36199378967285 (23.185912265179468) + | > loss_duration: 1.4654037952423096 (1.4250300775836995) + | > amp_scaler: 512.0 (542.327526132404) + | > loss_1: 30.340198516845703 (31.793082954991508) + | > grad_norm_1: tensor(109.6238, device='cuda:0') (tensor(130.3786, device='cuda:0')) + | > current_lr_0: 0.00019972517181056292 + | > current_lr_1: 0.00019972517181056292 + | > step_time: 1.2663 (0.9452869842276752) + | > loader_time: 0.0224 (0.015511134360310092) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035329067960698556 (-2.461798647616746e-05) + | > avg_loss_disc: 2.7018046429816707 (-0.06263278900308888) + | > avg_loss_disc_real_0: 0.2263222060939099 (-0.022635248430231764) + | > avg_loss_disc_real_1: 0.22223131897601675 (+0.0292418434264812) + | > avg_loss_disc_real_2: 0.2322035885237633 (+0.013423008804625658) + | > avg_loss_disc_real_3: 0.1994686538868762 (-0.014386012516123176) + | > avg_loss_disc_real_4: 0.24613106948264102 (+0.03051370065262976) + | > avg_loss_disc_real_5: 0.23992229110382973 (-0.010887065466414159) + | > avg_loss_0: 2.7018046429816707 (-0.06263278900308888) + | > avg_loss_gen: 1.8085014135279553 (+0.04242083113244233) + | > avg_loss_kl: 2.4830735896496066 (-0.05292703243012076) + | > avg_loss_feat: 2.749349467297818 (-0.06184500329037945) + | > avg_loss_mel: 22.684779187466237 (-0.5614565179703064) + | > avg_loss_duration: 1.412746069279123 (-0.0014494936516937784) + | > avg_loss_1: 31.138449526847676 (-0.6352573557102978) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_7092.pth + + > EPOCH: 13/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 04:46:45)  + + --> TIME: 2024-06-25 04:46:54 -- STEP: 8/591 -- GLOBAL_STEP: 7100 + | > loss_disc: 2.722562074661255 (2.692598342895508) + | > loss_disc_real_0: 0.18210043013095856 (0.213733347132802) + | > loss_disc_real_1: 0.23140941560268402 (0.22732057608664036) + | > loss_disc_real_2: 0.2370736002922058 (0.23257232829928398) + | > loss_disc_real_3: 0.2544441819190979 (0.23530437052249908) + | > loss_disc_real_4: 0.21330572664737701 (0.2307052444666624) + | > loss_disc_real_5: 0.21542693674564362 (0.23122194223105907) + | > loss_0: 2.722562074661255 (2.692598342895508) + | > grad_norm_0: tensor(7.6763, device='cuda:0') (tensor(8.9003, device='cuda:0')) + | > loss_gen: 1.9182275533676147 (1.8927850872278214) + | > loss_kl: 2.579458713531494 (2.631780654191971) + | > loss_feat: 3.178880214691162 (3.173821359872818) + | > loss_mel: 22.72403907775879 (23.071237802505493) + | > loss_duration: 1.3718891143798828 (1.3873380571603775) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.77249526977539 (32.156962871551514) + | > grad_norm_1: tensor(66.6845, device='cuda:0') (tensor(143.9897, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.7993 (0.7760011553764343) + | > loader_time: 0.0084 (0.00839194655418396) + + + --> TIME: 2024-06-25 04:47:14 -- STEP: 33/591 -- GLOBAL_STEP: 7125 + | > loss_disc: 2.742936849594116 (2.721111608274055) + | > loss_disc_real_0: 0.16469305753707886 (0.21876373729019455) + | > loss_disc_real_1: 0.23661641776561737 (0.22654620399980835) + | > loss_disc_real_2: 0.2846241593360901 (0.23467044712919177) + | > loss_disc_real_3: 0.2446494996547699 (0.23498273753758633) + | > loss_disc_real_4: 0.27806347608566284 (0.2346105525890986) + | > loss_disc_real_5: 0.28255343437194824 (0.23768708245320755) + | > loss_0: 2.742936849594116 (2.721111608274055) + | > grad_norm_0: tensor(11.9353, device='cuda:0') (tensor(11.8624, device='cuda:0')) + | > loss_gen: 1.7867929935455322 (1.8955542607740923) + | > loss_kl: 2.422913074493408 (2.5769583889932344) + | > loss_feat: 2.873471736907959 (3.0759040370131983) + | > loss_mel: 22.145715713500977 (23.086838057546906) + | > loss_duration: 1.409354567527771 (1.3821252367713235) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.638248443603516 (32.01737999193596) + | > grad_norm_1: tensor(146.6418, device='cuda:0') (tensor(138.1910, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.7875 (0.787360364740545) + | > loader_time: 0.0095 (0.009364084763960407) + + + --> TIME: 2024-06-25 04:47:35 -- STEP: 58/591 -- GLOBAL_STEP: 7150 + | > loss_disc: 2.7935843467712402 (2.725212269815905) + | > loss_disc_real_0: 0.19649654626846313 (0.2163325112698407) + | > loss_disc_real_1: 0.20118242502212524 (0.2272061782664266) + | > loss_disc_real_2: 0.28011709451675415 (0.23505719468511385) + | > loss_disc_real_3: 0.27337244153022766 (0.23495195189426685) + | > loss_disc_real_4: 0.25281980633735657 (0.23431782398758264) + | > loss_disc_real_5: 0.2570808529853821 (0.23734105249931073) + | > loss_0: 2.7935843467712402 (2.725212269815905) + | > grad_norm_0: tensor(5.6887, device='cuda:0') (tensor(10.5427, device='cuda:0')) + | > loss_gen: 1.8352452516555786 (1.883601227710987) + | > loss_kl: 2.4486069679260254 (2.546812127376425) + | > loss_feat: 2.9427690505981445 (3.0443214383618584) + | > loss_mel: 23.59356117248535 (23.04851383998476) + | > loss_duration: 1.3994739055633545 (1.3940693201689882) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.21965789794922 (31.917318015263) + | > grad_norm_1: tensor(88.6947, device='cuda:0') (tensor(127.8464, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.8274 (0.7958390753844689) + | > loader_time: 0.01 (0.009799472216902107) + + + --> TIME: 2024-06-25 04:47:56 -- STEP: 83/591 -- GLOBAL_STEP: 7175 + | > loss_disc: 2.7194504737854004 (2.7305986306753502) + | > loss_disc_real_0: 0.20694556832313538 (0.2169637527451458) + | > loss_disc_real_1: 0.19837185740470886 (0.22797465934810868) + | > loss_disc_real_2: 0.200378879904747 (0.23522079152515135) + | > loss_disc_real_3: 0.25814351439476013 (0.23461027497268586) + | > loss_disc_real_4: 0.21841877698898315 (0.23400619751717672) + | > loss_disc_real_5: 0.24654021859169006 (0.23778161011546492) + | > loss_0: 2.7194504737854004 (2.7305986306753502) + | > grad_norm_0: tensor(6.1630, device='cuda:0') (tensor(11.0686, device='cuda:0')) + | > loss_gen: 2.0761196613311768 (1.8837559596601738) + | > loss_kl: 2.3949790000915527 (2.5308207488921757) + | > loss_feat: 3.0902843475341797 (3.024582658905581) + | > loss_mel: 22.7493896484375 (23.057100962443524) + | > loss_duration: 1.4182884693145752 (1.4021051676876573) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.729063034057617 (31.89836557227445) + | > grad_norm_1: tensor(153.1856, device='cuda:0') (tensor(133.6737, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.8316 (0.8033263309892401) + | > loader_time: 0.011 (0.010153017848371028) + + + --> TIME: 2024-06-25 04:48:17 -- STEP: 108/591 -- GLOBAL_STEP: 7200 + | > loss_disc: 3.0597362518310547 (2.743365373876361) + | > loss_disc_real_0: 0.28286415338516235 (0.2188634737222283) + | > loss_disc_real_1: 0.2626364827156067 (0.23161394687162506) + | > loss_disc_real_2: 0.2874239385128021 (0.23719511539847762) + | > loss_disc_real_3: 0.32530125975608826 (0.2353047351870272) + | > loss_disc_real_4: 0.2183343917131424 (0.23476373266290734) + | > loss_disc_real_5: 0.24373935163021088 (0.23709929430926288) + | > loss_0: 3.0597362518310547 (2.743365373876361) + | > grad_norm_0: tensor(11.2396, device='cuda:0') (tensor(10.7782, device='cuda:0')) + | > loss_gen: 1.7356332540512085 (1.8933079220630504) + | > loss_kl: 2.4581732749938965 (2.515208734406366) + | > loss_feat: 2.828329563140869 (3.0352917512257895) + | > loss_mel: 23.887025833129883 (23.132142190580016) + | > loss_duration: 1.4475988149642944 (1.4077883285504795) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.35676193237305 (31.983739058176678) + | > grad_norm_1: tensor(114.0778, device='cuda:0') (tensor(148.9787, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.8467 (0.8116521879478737) + | > loader_time: 0.0121 (0.010510950176804158) + + + --> TIME: 2024-06-25 04:48:39 -- STEP: 133/591 -- GLOBAL_STEP: 7225 + | > loss_disc: 2.961921215057373 (2.77097006847984) + | > loss_disc_real_0: 0.21904301643371582 (0.21825854704344183) + | > loss_disc_real_1: 0.32173052430152893 (0.24224069535284115) + | > loss_disc_real_2: 0.2911035716533661 (0.2387485609466868) + | > loss_disc_real_3: 0.24528095126152039 (0.23643374622316288) + | > loss_disc_real_4: 0.26897138357162476 (0.23916030109376835) + | > loss_disc_real_5: 0.2261195033788681 (0.23807800690034278) + | > loss_0: 2.961921215057373 (2.77097006847984) + | > grad_norm_0: tensor(4.8775, device='cuda:0') (tensor(10.3907, device='cuda:0')) + | > loss_gen: 1.871668815612793 (1.8959827046645314) + | > loss_kl: 2.5272700786590576 (2.5053877292719107) + | > loss_feat: 2.459535598754883 (3.002938161218973) + | > loss_mel: 23.111398696899414 (23.230305248633364) + | > loss_duration: 1.4036049842834473 (1.409081025231153) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.37347984313965 (32.04369500525913) + | > grad_norm_1: tensor(102.7333, device='cuda:0') (tensor(151.9421, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.8318 (0.8182440592830342) + | > loader_time: 0.0115 (0.010725128919558421) + + + --> TIME: 2024-06-25 04:49:00 -- STEP: 158/591 -- GLOBAL_STEP: 7250 + | > loss_disc: 2.7315621376037598 (2.7842877273318147) + | > loss_disc_real_0: 0.16392427682876587 (0.21969411757927906) + | > loss_disc_real_1: 0.22361576557159424 (0.24641394511431078) + | > loss_disc_real_2: 0.2361193746328354 (0.23968785473063023) + | > loss_disc_real_3: 0.25819265842437744 (0.23672760702386686) + | > loss_disc_real_4: 0.20257478952407837 (0.239726749873614) + | > loss_disc_real_5: 0.2458687573671341 (0.23814629404982435) + | > loss_0: 2.7315621376037598 (2.7842877273318147) + | > grad_norm_0: tensor(10.0499, device='cuda:0') (tensor(10.5612, device='cuda:0')) + | > loss_gen: 1.9647899866104126 (1.8817324479923974) + | > loss_kl: 2.3757901191711426 (2.4945473701139034) + | > loss_feat: 2.882080316543579 (2.940271133108985) + | > loss_mel: 23.330820083618164 (23.20123371896864) + | > loss_duration: 1.398294448852539 (1.4071123486832724) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.9517765045166 (31.92489716976505) + | > grad_norm_1: tensor(133.0860, device='cuda:0') (tensor(147.1739, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.8457 (0.824214535423472) + | > loader_time: 0.0113 (0.01089730292935915) + + + --> TIME: 2024-06-25 04:49:22 -- STEP: 183/591 -- GLOBAL_STEP: 7275 + | > loss_disc: 2.6595306396484375 (2.779927683658289) + | > loss_disc_real_0: 0.18556489050388336 (0.2198107166857016) + | > loss_disc_real_1: 0.18720191717147827 (0.2442649093970575) + | > loss_disc_real_2: 0.19694261252880096 (0.23906322597154503) + | > loss_disc_real_3: 0.22605754435062408 (0.236135026796268) + | > loss_disc_real_4: 0.2253066450357437 (0.23985344537946043) + | > loss_disc_real_5: 0.19689826667308807 (0.23822481718545402) + | > loss_0: 2.6595306396484375 (2.779927683658289) + | > grad_norm_0: tensor(5.5171, device='cuda:0') (tensor(10.4677, device='cuda:0')) + | > loss_gen: 1.899531602859497 (1.8755166693463352) + | > loss_kl: 2.532443046569824 (2.4896362518352224) + | > loss_feat: 2.997971296310425 (2.924901103712822) + | > loss_mel: 23.145503997802734 (23.158244127784272) + | > loss_duration: 1.4136290550231934 (1.4059081416312462) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.98908042907715 (31.85420641873053) + | > grad_norm_1: tensor(80.9998, device='cuda:0') (tensor(139.3599, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.8593 (0.8295923973041806) + | > loader_time: 0.0114 (0.011020061096858462) + + + --> TIME: 2024-06-25 04:49:45 -- STEP: 208/591 -- GLOBAL_STEP: 7300 + | > loss_disc: 2.7469820976257324 (2.77405742957042) + | > loss_disc_real_0: 0.14988279342651367 (0.21956737210544255) + | > loss_disc_real_1: 0.22086039185523987 (0.24218721143328226) + | > loss_disc_real_2: 0.21916912496089935 (0.2385289782228378) + | > loss_disc_real_3: 0.2025379091501236 (0.23588270643869272) + | > loss_disc_real_4: 0.22957709431648254 (0.23906086865239418) + | > loss_disc_real_5: 0.20818457007408142 (0.2382297828220404) + | > loss_0: 2.7469820976257324 (2.77405742957042) + | > grad_norm_0: tensor(13.3794, device='cuda:0') (tensor(10.5381, device='cuda:0')) + | > loss_gen: 1.9188374280929565 (1.8748375509793942) + | > loss_kl: 2.4035212993621826 (2.4846384720160413) + | > loss_feat: 2.8899013996124268 (2.9252574867927112) + | > loss_mel: 23.120241165161133 (23.145497331252464) + | > loss_duration: 1.4210879802703857 (1.4056586474180222) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.75358772277832 (31.835889651225173) + | > grad_norm_1: tensor(72.2738, device='cuda:0') (tensor(133.4166, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.8729 (0.8345111551193091) + | > loader_time: 0.0137 (0.011193237625635592) + + + --> TIME: 2024-06-25 04:50:07 -- STEP: 233/591 -- GLOBAL_STEP: 7325 + | > loss_disc: 2.744051694869995 (2.7714784145355242) + | > loss_disc_real_0: 0.2378409206867218 (0.22009712201842935) + | > loss_disc_real_1: 0.23230724036693573 (0.2404902744958329) + | > loss_disc_real_2: 0.24958749115467072 (0.23841980694445417) + | > loss_disc_real_3: 0.24621978402137756 (0.2359354236990597) + | > loss_disc_real_4: 0.23712261021137238 (0.23875236293788632) + | > loss_disc_real_5: 0.23826318979263306 (0.2381818223536782) + | > loss_0: 2.744051694869995 (2.7714784145355242) + | > grad_norm_0: tensor(5.8636, device='cuda:0') (tensor(10.7252, device='cuda:0')) + | > loss_gen: 2.000514268875122 (1.874329011327719) + | > loss_kl: 2.4613451957702637 (2.4849202735229623) + | > loss_feat: 2.7682104110717773 (2.9229544570006016) + | > loss_mel: 22.813955307006836 (23.139783769206428) + | > loss_duration: 1.3997135162353516 (1.4053403159579494) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.44373893737793 (31.827327982038906) + | > grad_norm_1: tensor(89.0107, device='cuda:0') (tensor(130.9488, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.8694 (0.8391766425366054) + | > loader_time: 0.0156 (0.011501067697746052) + + + --> TIME: 2024-06-25 04:50:30 -- STEP: 258/591 -- GLOBAL_STEP: 7350 + | > loss_disc: 2.68788743019104 (2.769409974416099) + | > loss_disc_real_0: 0.16437917947769165 (0.22012047657324363) + | > loss_disc_real_1: 0.22338856756687164 (0.23922805568968603) + | > loss_disc_real_2: 0.21702954173088074 (0.23815519195194393) + | > loss_disc_real_3: 0.2040797770023346 (0.23596311938161998) + | > loss_disc_real_4: 0.287540078163147 (0.2386568457812302) + | > loss_disc_real_5: 0.28915777802467346 (0.2383542659208756) + | > loss_0: 2.68788743019104 (2.769409974416099) + | > grad_norm_0: tensor(9.4124, device='cuda:0') (tensor(10.7385, device='cuda:0')) + | > loss_gen: 1.9502712488174438 (1.8730060217916504) + | > loss_kl: 2.6010472774505615 (2.486057310141334) + | > loss_feat: 3.0775609016418457 (2.921876220740089) + | > loss_mel: 23.45921516418457 (23.136205015256422) + | > loss_duration: 1.3976666927337646 (1.4050259950549104) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.48575973510742 (31.822170752887587) + | > grad_norm_1: tensor(109.7138, device='cuda:0') (tensor(128.3683, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.8983 (0.84518175716548) + | > loader_time: 0.0134 (0.011796369109042859) + + + --> TIME: 2024-06-25 04:50:53 -- STEP: 283/591 -- GLOBAL_STEP: 7375 + | > loss_disc: 2.7566587924957275 (2.7667206049807938) + | > loss_disc_real_0: 0.16796758770942688 (0.21983924721344622) + | > loss_disc_real_1: 0.19592487812042236 (0.2382307141071495) + | > loss_disc_real_2: 0.29454708099365234 (0.23803271937707288) + | > loss_disc_real_3: 0.1996966451406479 (0.2360259654665162) + | > loss_disc_real_4: 0.21255992352962494 (0.2382606589646727) + | > loss_disc_real_5: 0.2518766224384308 (0.23820326716866172) + | > loss_0: 2.7566587924957275 (2.7667206049807938) + | > grad_norm_0: tensor(11.3669, device='cuda:0') (tensor(10.6138, device='cuda:0')) + | > loss_gen: 1.7523269653320312 (1.8721941903286183) + | > loss_kl: 2.5260250568389893 (2.486006891769571) + | > loss_feat: 2.8546359539031982 (2.9222597764153364) + | > loss_mel: 22.66510009765625 (23.122724027599972) + | > loss_duration: 1.3986124992370605 (1.405521984656371) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.196701049804688 (31.808707078859584) + | > grad_norm_1: tensor(132.4586, device='cuda:0') (tensor(130.9393, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.9118 (0.8504288870538501) + | > loader_time: 0.0159 (0.012088990464227359) + + + --> TIME: 2024-06-25 04:51:17 -- STEP: 308/591 -- GLOBAL_STEP: 7400 + | > loss_disc: 2.732414960861206 (2.766223432955805) + | > loss_disc_real_0: 0.34032994508743286 (0.22040564361233989) + | > loss_disc_real_1: 0.22562098503112793 (0.23760519002552155) + | > loss_disc_real_2: 0.24559004604816437 (0.2377426986667243) + | > loss_disc_real_3: 0.29196566343307495 (0.23612478012581925) + | > loss_disc_real_4: 0.24144449830055237 (0.23802950025185363) + | > loss_disc_real_5: 0.21921268105506897 (0.23803426775258857) + | > loss_0: 2.732414960861206 (2.766223432955805) + | > grad_norm_0: tensor(21.7905, device='cuda:0') (tensor(10.6995, device='cuda:0')) + | > loss_gen: 1.9311888217926025 (1.8709434510825516) + | > loss_kl: 2.5642952919006348 (2.4845762036063457) + | > loss_feat: 3.0331690311431885 (2.91494220495224) + | > loss_mel: 23.422496795654297 (23.107443914785016) + | > loss_duration: 1.3757712841033936 (1.4058739941615563) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.32692337036133 (31.783780005071076) + | > grad_norm_1: tensor(203.9824, device='cuda:0') (tensor(131.8899, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.9027 (0.8566178787838329) + | > loader_time: 0.0157 (0.012372224361865556) + + + --> TIME: 2024-06-25 04:51:41 -- STEP: 333/591 -- GLOBAL_STEP: 7425 + | > loss_disc: 2.7771525382995605 (2.7651718235588647) + | > loss_disc_real_0: 0.2036132514476776 (0.22014920133817661) + | > loss_disc_real_1: 0.24068695306777954 (0.2371369347915993) + | > loss_disc_real_2: 0.23126745223999023 (0.23781678827854247) + | > loss_disc_real_3: 0.20643877983093262 (0.23608365856312416) + | > loss_disc_real_4: 0.23937785625457764 (0.23777281240121023) + | > loss_disc_real_5: 0.2923904359340668 (0.23824577091692445) + | > loss_0: 2.7771525382995605 (2.7651718235588647) + | > grad_norm_0: tensor(4.1533, device='cuda:0') (tensor(10.6106, device='cuda:0')) + | > loss_gen: 1.7581647634506226 (1.869780307775503) + | > loss_kl: 2.407280921936035 (2.4820238469957245) + | > loss_feat: 2.76947283744812 (2.912059174643623) + | > loss_mel: 22.769222259521484 (23.092101758664793) + | > loss_duration: 1.4274671077728271 (1.406544596225292) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.131608963012695 (31.762509870099596) + | > grad_norm_1: tensor(175.4369, device='cuda:0') (tensor(130.4062, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.9285 (0.8621666660537949) + | > loader_time: 0.0149 (0.012596432511154962) + + + --> TIME: 2024-06-25 04:52:05 -- STEP: 358/591 -- GLOBAL_STEP: 7450 + | > loss_disc: 2.786092519760132 (2.7647832738620606) + | > loss_disc_real_0: 0.14474526047706604 (0.2200918303771392) + | > loss_disc_real_1: 0.21537676453590393 (0.2366182076115182) + | > loss_disc_real_2: 0.25960004329681396 (0.23813890890535697) + | > loss_disc_real_3: 0.28250646591186523 (0.23609838505696984) + | > loss_disc_real_4: 0.24945306777954102 (0.2377374031107519) + | > loss_disc_real_5: 0.24548351764678955 (0.23820819364413204) + | > loss_0: 2.786092519760132 (2.7647832738620606) + | > grad_norm_0: tensor(15.4747, device='cuda:0') (tensor(10.7604, device='cuda:0')) + | > loss_gen: 1.886348009109497 (1.8685535841813967) + | > loss_kl: 2.5081729888916016 (2.4795808698877946) + | > loss_feat: 2.9031381607055664 (2.9074778516865316) + | > loss_mel: 23.097238540649414 (23.084096684802176) + | > loss_duration: 1.4631288051605225 (1.4080660736094646) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.8580265045166 (31.7477751950312) + | > grad_norm_1: tensor(119.6370, device='cuda:0') (tensor(130.2686, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.9229 (0.8681009062175644) + | > loader_time: 0.0146 (0.012843614183990656) + + + --> TIME: 2024-06-25 04:52:29 -- STEP: 383/591 -- GLOBAL_STEP: 7475 + | > loss_disc: 2.726705312728882 (2.7637502020395144) + | > loss_disc_real_0: 0.14857006072998047 (0.2202799450946539) + | > loss_disc_real_1: 0.2442682385444641 (0.23630510519596679) + | > loss_disc_real_2: 0.2676956355571747 (0.2380221720697363) + | > loss_disc_real_3: 0.2502186596393585 (0.23604570259157734) + | > loss_disc_real_4: 0.23859743773937225 (0.23754627356796912) + | > loss_disc_real_5: 0.23284079134464264 (0.23810914637370148) + | > loss_0: 2.726705312728882 (2.7637502020395144) + | > grad_norm_0: tensor(14.4747, device='cuda:0') (tensor(10.7834, device='cuda:0')) + | > loss_gen: 1.7945220470428467 (1.867989507729951) + | > loss_kl: 2.5619988441467285 (2.4780906971067425) + | > loss_feat: 3.153754949569702 (2.9043707760444826) + | > loss_mel: 22.891878128051758 (23.052723742651878) + | > loss_duration: 1.432117223739624 (1.409362077401762) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.834270477294922 (31.71253691391908) + | > grad_norm_1: tensor(185.9985, device='cuda:0') (tensor(130.7982, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.9791 (0.8743966428791574) + | > loader_time: 0.0159 (0.013121380195916485) + + + --> TIME: 2024-06-25 04:52:54 -- STEP: 408/591 -- GLOBAL_STEP: 7500 + | > loss_disc: 2.763735771179199 (2.762562795012605) + | > loss_disc_real_0: 0.28262582421302795 (0.22035263537191876) + | > loss_disc_real_1: 0.2210833877325058 (0.23557289419513122) + | > loss_disc_real_2: 0.2247711718082428 (0.23783018063826888) + | > loss_disc_real_3: 0.2610894739627838 (0.235937310638381) + | > loss_disc_real_4: 0.2033722847700119 (0.2374525703855005) + | > loss_disc_real_5: 0.20996859669685364 (0.23805197616856472) + | > loss_0: 2.763735771179199 (2.762562795012605) + | > grad_norm_0: tensor(13.1246, device='cuda:0') (tensor(10.7197, device='cuda:0')) + | > loss_gen: 1.8315633535385132 (1.8668328105818992) + | > loss_kl: 2.5130274295806885 (2.47624237163394) + | > loss_feat: 2.99507474899292 (2.901210014726602) + | > loss_mel: 22.671688079833984 (23.033625658820657) + | > loss_duration: 1.4258986711502075 (1.4101603814199866) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.437253952026367 (31.68807136311251) + | > grad_norm_1: tensor(95.4812, device='cuda:0') (tensor(129.1545, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 0.9745 (0.880375922310586) + | > loader_time: 0.0189 (0.013393680838977596) + + + --> TIME: 2024-06-25 04:53:20 -- STEP: 433/591 -- GLOBAL_STEP: 7525 + | > loss_disc: 2.8046786785125732 (2.7618213459746115) + | > loss_disc_real_0: 0.15984675288200378 (0.22019167610940976) + | > loss_disc_real_1: 0.26354357600212097 (0.23532700063725268) + | > loss_disc_real_2: 0.2566576600074768 (0.23786342629246568) + | > loss_disc_real_3: 0.135492742061615 (0.23579073193166877) + | > loss_disc_real_4: 0.2163124531507492 (0.23738491776083137) + | > loss_disc_real_5: 0.19692543148994446 (0.23813160417674595) + | > loss_0: 2.8046786785125732 (2.7618213459746115) + | > grad_norm_0: tensor(16.9163, device='cuda:0') (tensor(10.8803, device='cuda:0')) + | > loss_gen: 2.0214686393737793 (1.8676275571003798) + | > loss_kl: 2.391268253326416 (2.4742088978615415) + | > loss_feat: 2.739576578140259 (2.8999798655785263) + | > loss_mel: 22.589250564575195 (23.01368524846524) + | > loss_duration: 1.4325246810913086 (1.4109780190998638) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.174091339111328 (31.666479723007523) + | > grad_norm_1: tensor(194.7016, device='cuda:0') (tensor(130.9194, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 1.005 (0.8871696336715381) + | > loader_time: 0.0195 (0.01364328128911607) + + + --> TIME: 2024-06-25 04:53:46 -- STEP: 458/591 -- GLOBAL_STEP: 7550 + | > loss_disc: 2.6866939067840576 (2.7611660759521883) + | > loss_disc_real_0: 0.12554757297039032 (0.22015264384676275) + | > loss_disc_real_1: 0.20577001571655273 (0.23488748314339) + | > loss_disc_real_2: 0.22774098813533783 (0.23789535427067476) + | > loss_disc_real_3: 0.20295412838459015 (0.23622165132297698) + | > loss_disc_real_4: 0.20221790671348572 (0.23723640016332984) + | > loss_disc_real_5: 0.22801725566387177 (0.23800061210404316) + | > loss_0: 2.6866939067840576 (2.7611660759521883) + | > grad_norm_0: tensor(22.3317, device='cuda:0') (tensor(10.8904, device='cuda:0')) + | > loss_gen: 2.0515201091766357 (1.8669008006175012) + | > loss_kl: 2.515547037124634 (2.4724773590221147) + | > loss_feat: 2.7349677085876465 (2.895508569921469) + | > loss_mel: 21.81593894958496 (22.993422945513995) + | > loss_duration: 1.4421536922454834 (1.4118539675354442) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.56012725830078 (31.640163767285745) + | > grad_norm_1: tensor(151.1488, device='cuda:0') (tensor(130.7547, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 1.0164 (0.8941881333896687) + | > loader_time: 0.0159 (0.013908229019964632) + + + --> TIME: 2024-06-25 04:54:12 -- STEP: 483/591 -- GLOBAL_STEP: 7575 + | > loss_disc: 2.741100549697876 (2.7620709909909014) + | > loss_disc_real_0: 0.19451171159744263 (0.22073771619463559) + | > loss_disc_real_1: 0.202703595161438 (0.23470554166082022) + | > loss_disc_real_2: 0.24176114797592163 (0.23784641662369604) + | > loss_disc_real_3: 0.20002661645412445 (0.2361583221649778) + | > loss_disc_real_4: 0.2382044941186905 (0.23732689260572626) + | > loss_disc_real_5: 0.23636460304260254 (0.2381217229502048) + | > loss_0: 2.741100549697876 (2.7620709909909014) + | > grad_norm_0: tensor(5.5687, device='cuda:0') (tensor(10.9912, device='cuda:0')) + | > loss_gen: 1.8239599466323853 (1.8658402442438509) + | > loss_kl: 2.458728551864624 (2.4706904508065475) + | > loss_feat: 2.7513673305511475 (2.8887627791173713) + | > loss_mel: 22.382509231567383 (22.981465515389452) + | > loss_duration: 1.4587159156799316 (1.413165570539471) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.875280380249023 (31.619924679552796) + | > grad_norm_1: tensor(100.1854, device='cuda:0') (tensor(132.3639, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 1.0437 (0.9017468441593968) + | > loader_time: 0.0167 (0.014160448966806231) + + + --> TIME: 2024-06-25 04:54:40 -- STEP: 508/591 -- GLOBAL_STEP: 7600 + | > loss_disc: 2.7167067527770996 (2.7612058839460056) + | > loss_disc_real_0: 0.29487377405166626 (0.2208354949335179) + | > loss_disc_real_1: 0.22011087834835052 (0.23442366373116577) + | > loss_disc_real_2: 0.21931077539920807 (0.23783303685779647) + | > loss_disc_real_3: 0.2324497103691101 (0.23616272070276456) + | > loss_disc_real_4: 0.2722737789154053 (0.23714273854151485) + | > loss_disc_real_5: 0.24807345867156982 (0.23808382117138135) + | > loss_0: 2.7167067527770996 (2.7612058839460056) + | > grad_norm_0: tensor(11.8253, device='cuda:0') (tensor(10.9624, device='cuda:0')) + | > loss_gen: 1.7013633251190186 (1.86511214511601) + | > loss_kl: 2.4604530334472656 (2.468472692440813) + | > loss_feat: 2.9738636016845703 (2.885784086279982) + | > loss_mel: 22.641132354736328 (22.9609145141962) + | > loss_duration: 1.4076873064041138 (1.4144529019753769) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.184499740600586 (31.59473646719625) + | > grad_norm_1: tensor(127.4789, device='cuda:0') (tensor(130.5957, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 1.0978 (0.9111043036453367) + | > loader_time: 0.0252 (0.014424149914989326) + + + --> TIME: 2024-06-25 04:55:10 -- STEP: 533/591 -- GLOBAL_STEP: 7625 + | > loss_disc: 2.7121050357818604 (2.760541061523037) + | > loss_disc_real_0: 0.2456679344177246 (0.22062508833173516) + | > loss_disc_real_1: 0.23249520361423492 (0.23423031595925228) + | > loss_disc_real_2: 0.20094789564609528 (0.2376504400657668) + | > loss_disc_real_3: 0.22734996676445007 (0.23608981014192665) + | > loss_disc_real_4: 0.23681895434856415 (0.23710054097658692) + | > loss_disc_real_5: 0.2405652403831482 (0.23806765236617478) + | > loss_0: 2.7121050357818604 (2.760541061523037) + | > grad_norm_0: tensor(11.1191, device='cuda:0') (tensor(10.9231, device='cuda:0')) + | > loss_gen: 1.7796545028686523 (1.864202396730992) + | > loss_kl: 2.460115909576416 (2.466178635047926) + | > loss_feat: 2.572258710861206 (2.8810109152802834) + | > loss_mel: 21.596784591674805 (22.933836144309552) + | > loss_duration: 1.4211945533752441 (1.4159870668826366) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.830007553100586 (31.561215277237025) + | > grad_norm_1: tensor(147.7083, device='cuda:0') (tensor(129.3204, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 1.3264 (0.9231040334612075) + | > loader_time: 0.0201 (0.014717568152393683) + + + --> TIME: 2024-06-25 04:55:42 -- STEP: 558/591 -- GLOBAL_STEP: 7650 + | > loss_disc: 2.80535626411438 (2.7605555279707823) + | > loss_disc_real_0: 0.4738120436668396 (0.2208391732292577) + | > loss_disc_real_1: 0.22071848809719086 (0.2340208986593831) + | > loss_disc_real_2: 0.25208014249801636 (0.23762055182008332) + | > loss_disc_real_3: 0.23008109629154205 (0.23628952511749815) + | > loss_disc_real_4: 0.23725056648254395 (0.23717908849639271) + | > loss_disc_real_5: 0.23872797191143036 (0.23804728665445868) + | > loss_0: 2.80535626411438 (2.7605555279707823) + | > grad_norm_0: tensor(40.4573, device='cuda:0') (tensor(11.0778, device='cuda:0')) + | > loss_gen: 1.6591322422027588 (1.8639979537669904) + | > loss_kl: 2.454373598098755 (2.464097265274293) + | > loss_feat: 2.7815418243408203 (2.876561865584398) + | > loss_mel: 21.994152069091797 (22.91471028071578) + | > loss_duration: 1.4134315252304077 (1.417471203752744) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.302631378173828 (31.536838675058018) + | > grad_norm_1: tensor(80.0274, device='cuda:0') (tensor(129.8208, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 1.217 (0.9371263318591647) + | > loader_time: 0.0199 (0.015031834657046963) + + + --> TIME: 2024-06-25 04:56:13 -- STEP: 583/591 -- GLOBAL_STEP: 7675 + | > loss_disc: 2.684567928314209 (2.7601365114771483) + | > loss_disc_real_0: 0.24267658591270447 (0.22073610235599875) + | > loss_disc_real_1: 0.2594176232814789 (0.2338197356022856) + | > loss_disc_real_2: 0.24003541469573975 (0.23754082852734915) + | > loss_disc_real_3: 0.2446514368057251 (0.23627889969132163) + | > loss_disc_real_4: 0.22473178803920746 (0.23707750472871145) + | > loss_disc_real_5: 0.25169166922569275 (0.23799180227937256) + | > loss_0: 2.684567928314209 (2.7601365114771483) + | > grad_norm_0: tensor(6.5050, device='cuda:0') (tensor(11.0119, device='cuda:0')) + | > loss_gen: 1.7501416206359863 (1.8632762716319509) + | > loss_kl: 2.4089369773864746 (2.4617864116387334) + | > loss_feat: 2.7363126277923584 (2.8718857319440896) + | > loss_mel: 22.403711318969727 (22.895126754996305) + | > loss_duration: 1.4439637660980225 (1.4189647902346398) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.743066787719727 (31.511040074134158) + | > grad_norm_1: tensor(190.7629, device='cuda:0') (tensor(128.5510, device='cuda:0')) + | > current_lr_0: 0.0001997002061640866 + | > current_lr_1: 0.0001997002061640866 + | > step_time: 1.1793 (0.949735141741064) + | > loader_time: 0.0206 (0.015379569133804354) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00357041967676041 (+3.75128806905543e-05) + | > avg_loss_disc: 2.7407472996001547 (+0.03894265661848406) + | > avg_loss_disc_real_0: 0.20080714339905598 (-0.02551506269485393) + | > avg_loss_disc_real_1: 0.21351255032610386 (-0.00871876864991289) + | > avg_loss_disc_real_2: 0.22362066298089128 (-0.008582925542872016) + | > avg_loss_disc_real_3: 0.1982725960143069 (-0.001196057872569295) + | > avg_loss_disc_real_4: 0.22041864154186655 (-0.025712427940774468) + | > avg_loss_disc_real_5: 0.2369356833873911 (-0.002986607716438633) + | > avg_loss_0: 2.7407472996001547 (+0.03894265661848406) + | > avg_loss_gen: 1.7016147248288418 (-0.10688668869911355) + | > avg_loss_kl: 2.4848686025497755 (+0.0017950129001689241) + | > avg_loss_feat: 2.8487686248535806 (+0.0994191575557628) + | > avg_loss_mel: 22.39269333697379 (-0.2920858504924482) + | > avg_loss_duration: 1.401727742337166 (-0.011018326941957035) + | > avg_loss_1: 30.829672874288356 (-0.3087766525593203) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_7683.pth + + > EPOCH: 14/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 04:56:36)  + + --> TIME: 2024-06-25 04:56:51 -- STEP: 17/591 -- GLOBAL_STEP: 7700 + | > loss_disc: 2.7619009017944336 (2.7312370328342213) + | > loss_disc_real_0: 0.21734564006328583 (0.22805367673144622) + | > loss_disc_real_1: 0.21788537502288818 (0.22210649006506977) + | > loss_disc_real_2: 0.2737830579280853 (0.2344013619072297) + | > loss_disc_real_3: 0.24818699061870575 (0.23893443626516006) + | > loss_disc_real_4: 0.26626214385032654 (0.23689512470189264) + | > loss_disc_real_5: 0.25426700711250305 (0.24056548143134399) + | > loss_0: 2.7619009017944336 (2.7312370328342213) + | > grad_norm_0: tensor(5.4474, device='cuda:0') (tensor(13.0662, device='cuda:0')) + | > loss_gen: 1.6344544887542725 (1.887075003455667) + | > loss_kl: 2.664562463760376 (2.660509025349337) + | > loss_feat: 2.7380950450897217 (3.1601622385137222) + | > loss_mel: 22.5410213470459 (22.926631590899298) + | > loss_duration: 1.3307148218154907 (1.373436773524565) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.908849716186523 (32.00781485613655) + | > grad_norm_1: tensor(72.8434, device='cuda:0') (tensor(116.7377, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.7795 (0.7769145124098834) + | > loader_time: 0.0092 (0.008979615043191348) + + + --> TIME: 2024-06-25 04:57:12 -- STEP: 42/591 -- GLOBAL_STEP: 7725 + | > loss_disc: 2.756056070327759 (2.738553830555507) + | > loss_disc_real_0: 0.25938355922698975 (0.22351282533435596) + | > loss_disc_real_1: 0.2923160195350647 (0.22590175022681555) + | > loss_disc_real_2: 0.21342192590236664 (0.2333716047661645) + | > loss_disc_real_3: 0.19409771263599396 (0.23651759113584245) + | > loss_disc_real_4: 0.24044328927993774 (0.23623719669523693) + | > loss_disc_real_5: 0.24286501109600067 (0.23828422384602682) + | > loss_0: 2.756056070327759 (2.738553830555507) + | > grad_norm_0: tensor(10.4334, device='cuda:0') (tensor(12.3252, device='cuda:0')) + | > loss_gen: 1.7243674993515015 (1.8781481271698361) + | > loss_kl: 2.358489751815796 (2.6063605036054347) + | > loss_feat: 2.92655611038208 (3.0642086381003972) + | > loss_mel: 22.912982940673828 (22.868869554428827) + | > loss_duration: 1.4028334617614746 (1.3827486889702933) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.32522964477539 (31.8003355662028) + | > grad_norm_1: tensor(62.2374, device='cuda:0') (tensor(111.4575, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.8051 (0.7886188484373546) + | > loader_time: 0.0101 (0.00950804210844494) + + + --> TIME: 2024-06-25 04:57:32 -- STEP: 67/591 -- GLOBAL_STEP: 7750 + | > loss_disc: 2.783564805984497 (2.7414958939623477) + | > loss_disc_real_0: 0.1936669945716858 (0.2202857805054579) + | > loss_disc_real_1: 0.1947016566991806 (0.2269263336462761) + | > loss_disc_real_2: 0.2514181435108185 (0.23547251856149132) + | > loss_disc_real_3: 0.16329997777938843 (0.23616680836499626) + | > loss_disc_real_4: 0.25001153349876404 (0.23565211580760442) + | > loss_disc_real_5: 0.2617942988872528 (0.23892423399348758) + | > loss_0: 2.783564805984497 (2.7414958939623477) + | > grad_norm_0: tensor(11.4299, device='cuda:0') (tensor(10.6976, device='cuda:0')) + | > loss_gen: 1.9902740716934204 (1.8787872417649227) + | > loss_kl: 2.424901247024536 (2.563400250762257) + | > loss_feat: 2.9097883701324463 (3.034829506233557) + | > loss_mel: 22.778518676757812 (22.873462477726722) + | > loss_duration: 1.4013911485671997 (1.3915679561558054) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.50487518310547 (31.742047551852554) + | > grad_norm_1: tensor(65.9750, device='cuda:0') (tensor(114.5008, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.8344 (0.798411319504923) + | > loader_time: 0.0122 (0.00995111109605476) + + + --> TIME: 2024-06-25 04:57:54 -- STEP: 92/591 -- GLOBAL_STEP: 7775 + | > loss_disc: 2.7201244831085205 (2.746133413003839) + | > loss_disc_real_0: 0.24676838517189026 (0.22187256740163203) + | > loss_disc_real_1: 0.2467663586139679 (0.22807693125113196) + | > loss_disc_real_2: 0.21916832029819489 (0.23563304025193918) + | > loss_disc_real_3: 0.2289029210805893 (0.2366513545098512) + | > loss_disc_real_4: 0.24951617419719696 (0.23590961876122848) + | > loss_disc_real_5: 0.26473915576934814 (0.23894568759462106) + | > loss_0: 2.7201244831085205 (2.746133413003839) + | > grad_norm_0: tensor(8.3542, device='cuda:0') (tensor(11.3703, device='cuda:0')) + | > loss_gen: 1.8233833312988281 (1.8759895096654478) + | > loss_kl: 2.5186309814453125 (2.546941438446876) + | > loss_feat: 3.2944841384887695 (3.02108830731848) + | > loss_mel: 24.176315307617188 (22.989360664201826) + | > loss_duration: 1.3885669708251953 (1.4007765287938327) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 33.201377868652344 (31.83415647175001) + | > grad_norm_1: tensor(179.3454, device='cuda:0') (tensor(117.6156, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.8264 (0.8069256414537844) + | > loader_time: 0.0115 (0.010411894839742907) + + + --> TIME: 2024-06-25 04:58:15 -- STEP: 117/591 -- GLOBAL_STEP: 7800 + | > loss_disc: 2.7702460289001465 (2.7462416245387153) + | > loss_disc_real_0: 0.17158964276313782 (0.22172280579295933) + | > loss_disc_real_1: 0.28192415833473206 (0.22855901985596389) + | > loss_disc_real_2: 0.24984167516231537 (0.23555091456470326) + | > loss_disc_real_3: 0.22308588027954102 (0.23634724433605486) + | > loss_disc_real_4: 0.24492017924785614 (0.23534890907442468) + | > loss_disc_real_5: 0.2411354035139084 (0.23816610783593267) + | > loss_0: 2.7702460289001465 (2.7462416245387153) + | > grad_norm_0: tensor(9.9448, device='cuda:0') (tensor(11.2891, device='cuda:0')) + | > loss_gen: 1.8118722438812256 (1.8729133575390546) + | > loss_kl: 2.31418776512146 (2.5202502959813837) + | > loss_feat: 2.8745059967041016 (3.0160255880437346) + | > loss_mel: 22.969600677490234 (22.984708818614997) + | > loss_duration: 1.380737543106079 (1.403357089075268) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.35090446472168 (31.79725518185868) + | > grad_norm_1: tensor(97.9235, device='cuda:0') (tensor(123.4426, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.8658 (0.8138228298252465) + | > loader_time: 0.0126 (0.010647396756033609) + + + --> TIME: 2024-06-25 04:58:37 -- STEP: 142/591 -- GLOBAL_STEP: 7825 + | > loss_disc: 2.808264970779419 (2.74962247089601) + | > loss_disc_real_0: 0.26314541697502136 (0.2214148183628707) + | > loss_disc_real_1: 0.188472181558609 (0.2291260691175998) + | > loss_disc_real_2: 0.24877065420150757 (0.23624066819607373) + | > loss_disc_real_3: 0.2476860135793686 (0.23645663523757962) + | > loss_disc_real_4: 0.25221502780914307 (0.23542803431480702) + | > loss_disc_real_5: 0.27119237184524536 (0.2389222147389197) + | > loss_0: 2.808264970779419 (2.74962247089601) + | > grad_norm_0: tensor(8.0440, device='cuda:0') (tensor(11.3430, device='cuda:0')) + | > loss_gen: 1.9016146659851074 (1.8757285870296854) + | > loss_kl: 2.428344488143921 (2.5115987633315613) + | > loss_feat: 2.7705349922180176 (3.0110206402523416) + | > loss_mel: 22.724468231201172 (22.977588465515996) + | > loss_duration: 1.4219284057617188 (1.4037494928064482) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.246891021728516 (31.779685933824997) + | > grad_norm_1: tensor(153.7522, device='cuda:0') (tensor(126.9583, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.8436 (0.8196033709485766) + | > loader_time: 0.0117 (0.010814925314674912) + + + --> TIME: 2024-06-25 04:58:58 -- STEP: 167/591 -- GLOBAL_STEP: 7850 + | > loss_disc: 2.653146266937256 (2.7476543209509954) + | > loss_disc_real_0: 0.14818958938121796 (0.22130404693816236) + | > loss_disc_real_1: 0.20707882940769196 (0.22865099687419252) + | > loss_disc_real_2: 0.25468340516090393 (0.2360403501166555) + | > loss_disc_real_3: 0.22918418049812317 (0.2364334252601612) + | > loss_disc_real_4: 0.2332913875579834 (0.23550652886579138) + | > loss_disc_real_5: 0.23318269848823547 (0.2381852098746214) + | > loss_0: 2.653146266937256 (2.7476543209509954) + | > grad_norm_0: tensor(11.7107, device='cuda:0') (tensor(11.4589, device='cuda:0')) + | > loss_gen: 1.9634159803390503 (1.8745159410431) + | > loss_kl: 2.5332822799682617 (2.5083690432017445) + | > loss_feat: 3.0754427909851074 (3.0108044161768017) + | > loss_mel: 22.687719345092773 (22.949214866775232) + | > loss_duration: 1.3578239679336548 (1.4017757063140412) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.61768341064453 (31.744679936391865) + | > grad_norm_1: tensor(73.2576, device='cuda:0') (tensor(126.8485, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.8802 (0.8255593805256004) + | > loader_time: 0.0125 (0.011028366888354638) + + + --> TIME: 2024-06-25 04:59:21 -- STEP: 192/591 -- GLOBAL_STEP: 7875 + | > loss_disc: 2.7346439361572266 (2.745832905173303) + | > loss_disc_real_0: 0.2290046215057373 (0.22088329039979726) + | > loss_disc_real_1: 0.23249505460262299 (0.22870062679673234) + | > loss_disc_real_2: 0.21574638783931732 (0.23578014496403435) + | > loss_disc_real_3: 0.2976994216442108 (0.2365901394902418) + | > loss_disc_real_4: 0.2638413906097412 (0.2353568485317131) + | > loss_disc_real_5: 0.22139832377433777 (0.23803097886654237) + | > loss_0: 2.7346439361572266 (2.745832905173303) + | > grad_norm_0: tensor(5.9416, device='cuda:0') (tensor(11.3545, device='cuda:0')) + | > loss_gen: 1.9015684127807617 (1.8750252332538366) + | > loss_kl: 2.6499106884002686 (2.507797052462895) + | > loss_feat: 3.0310323238372803 (3.011053767055273) + | > loss_mel: 22.40081024169922 (22.93116693695387) + | > loss_duration: 1.394870400428772 (1.4006370591620605) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.378192901611328 (31.72568001349767) + | > grad_norm_1: tensor(141.0555, device='cuda:0') (tensor(123.7782, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.859 (0.8309989633659521) + | > loader_time: 0.0117 (0.011172706882158911) + + + --> TIME: 2024-06-25 04:59:43 -- STEP: 217/591 -- GLOBAL_STEP: 7900 + | > loss_disc: 2.7528793811798096 (2.7454268734575975) + | > loss_disc_real_0: 0.2639966905117035 (0.22066812018095622) + | > loss_disc_real_1: 0.2368488609790802 (0.22861377269990982) + | > loss_disc_real_2: 0.22666116058826447 (0.2358464314366266) + | > loss_disc_real_3: 0.21016091108322144 (0.23628206942487973) + | > loss_disc_real_4: 0.21531124413013458 (0.23520804852384575) + | > loss_disc_real_5: 0.22214913368225098 (0.2382661246346988) + | > loss_0: 2.7528793811798096 (2.7454268734575975) + | > grad_norm_0: tensor(10.4146, device='cuda:0') (tensor(11.5072, device='cuda:0')) + | > loss_gen: 1.8863613605499268 (1.874239922119176) + | > loss_kl: 2.555830955505371 (2.501161680792883) + | > loss_feat: 3.013800859451294 (3.0059424710163873) + | > loss_mel: 22.42832374572754 (22.90624046325684) + | > loss_duration: 1.3616294860839844 (1.3998354810723512) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.245946884155273 (31.6874199528848) + | > grad_norm_1: tensor(106.5880, device='cuda:0') (tensor(126.5643, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.8658 (0.8358639033708705) + | > loader_time: 0.0142 (0.011388556748491274) + + + --> TIME: 2024-06-25 05:00:05 -- STEP: 242/591 -- GLOBAL_STEP: 7925 + | > loss_disc: 2.755953788757324 (2.744949910266341) + | > loss_disc_real_0: 0.3410513699054718 (0.22084419601712346) + | > loss_disc_real_1: 0.22719590365886688 (0.228707264708586) + | > loss_disc_real_2: 0.22371888160705566 (0.23570209513764737) + | > loss_disc_real_3: 0.2259061336517334 (0.2362348996410685) + | > loss_disc_real_4: 0.21979323029518127 (0.235160416623285) + | > loss_disc_real_5: 0.19837968051433563 (0.2381873096316314) + | > loss_0: 2.755953788757324 (2.744949910266341) + | > grad_norm_0: tensor(20.6985, device='cuda:0') (tensor(11.7982, device='cuda:0')) + | > loss_gen: 1.8463550806045532 (1.8754739052008007) + | > loss_kl: 2.468186616897583 (2.4953703062593444) + | > loss_feat: 3.0724282264709473 (3.0092669823938163) + | > loss_mel: 22.825265884399414 (22.907864074076507) + | > loss_duration: 1.448836326599121 (1.3997821068960774) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.66107177734375 (31.687757295025282) + | > grad_norm_1: tensor(161.7825, device='cuda:0') (tensor(130.2352, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.8888 (0.8410461776512713) + | > loader_time: 0.0136 (0.011647764316275096) + + + --> TIME: 2024-06-25 05:00:28 -- STEP: 267/591 -- GLOBAL_STEP: 7950 + | > loss_disc: 2.74358868598938 (2.7427071462409782) + | > loss_disc_real_0: 0.22294218838214874 (0.22007800404275402) + | > loss_disc_real_1: 0.2533562183380127 (0.22830067881930632) + | > loss_disc_real_2: 0.23114702105522156 (0.23573960380607775) + | > loss_disc_real_3: 0.21460853517055511 (0.2363197838099262) + | > loss_disc_real_4: 0.25300198793411255 (0.23533463053935477) + | > loss_disc_real_5: 0.26934996247291565 (0.23844301002972135) + | > loss_0: 2.74358868598938 (2.7427071462409782) + | > grad_norm_0: tensor(3.5426, device='cuda:0') (tensor(11.3641, device='cuda:0')) + | > loss_gen: 1.6961430311203003 (1.8743959732269972) + | > loss_kl: 2.457258462905884 (2.4952038816744917) + | > loss_feat: 2.9030051231384277 (3.0102342228764454) + | > loss_mel: 23.338085174560547 (22.881328861365162) + | > loss_duration: 1.4373891353607178 (1.3994320197944783) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.831880569458008 (31.660594868749268) + | > grad_norm_1: tensor(71.2605, device='cuda:0') (tensor(125.8388, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.8885 (0.8462845868385687) + | > loader_time: 0.0156 (0.011916430255446985) + + + --> TIME: 2024-06-25 05:00:52 -- STEP: 292/591 -- GLOBAL_STEP: 7975 + | > loss_disc: 2.9009454250335693 (2.743616568715606) + | > loss_disc_real_0: 0.059061504900455475 (0.2199391405263992) + | > loss_disc_real_1: 0.17221982777118683 (0.22787204042893566) + | > loss_disc_real_2: 0.22452351450920105 (0.23596063894155908) + | > loss_disc_real_3: 0.20984522998332977 (0.23621247759828828) + | > loss_disc_real_4: 0.21904581785202026 (0.23530135783430647) + | > loss_disc_real_5: 0.25622832775115967 (0.23835625264742602) + | > loss_0: 2.9009454250335693 (2.743616568715606) + | > grad_norm_0: tensor(47.1027, device='cuda:0') (tensor(11.5617, device='cuda:0')) + | > loss_gen: 1.881834864616394 (1.873981477463082) + | > loss_kl: 2.4821383953094482 (2.4918809133033224) + | > loss_feat: 2.9960036277770996 (3.00740023998365) + | > loss_mel: 22.56772232055664 (22.8642149102198) + | > loss_duration: 1.3927905559539795 (1.4000998167958976) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.32048797607422 (31.63757730510137) + | > grad_norm_1: tensor(99.5638, device='cuda:0') (tensor(122.9467, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.93 (0.8514867041208971) + | > loader_time: 0.014 (0.012153986382157833) + + + --> TIME: 2024-06-25 05:01:15 -- STEP: 317/591 -- GLOBAL_STEP: 8000 + | > loss_disc: 2.7402071952819824 (2.744009677544002) + | > loss_disc_real_0: 0.24529078602790833 (0.2203451477296721) + | > loss_disc_real_1: 0.2736206650733948 (0.22808905726163545) + | > loss_disc_real_2: 0.23188847303390503 (0.23586708227354644) + | > loss_disc_real_3: 0.21392777562141418 (0.2361400490303521) + | > loss_disc_real_4: 0.25183790922164917 (0.23532300317889132) + | > loss_disc_real_5: 0.2645815908908844 (0.2384516809264944) + | > loss_0: 2.7402071952819824 (2.744009677544002) + | > grad_norm_0: tensor(6.7253, device='cuda:0') (tensor(11.4113, device='cuda:0')) + | > loss_gen: 1.900459885597229 (1.8735237012526214) + | > loss_kl: 2.4335083961486816 (2.4921409865659125) + | > loss_feat: 3.032820224761963 (3.0028065913107116) + | > loss_mel: 23.084888458251953 (22.86904462579673) + | > loss_duration: 1.3989896774291992 (1.4004562705849246) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.850666046142578 (31.637972142794155) + | > grad_norm_1: tensor(172.1227, device='cuda:0') (tensor(124.8697, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.909 (0.8567352302442964) + | > loader_time: 0.0144 (0.012399433539116801) + + + --> TIME: 2024-06-25 05:01:39 -- STEP: 342/591 -- GLOBAL_STEP: 8025 + | > loss_disc: 2.7821238040924072 (2.74363538675141) + | > loss_disc_real_0: 0.1518460512161255 (0.22007640760544447) + | > loss_disc_real_1: 0.22464866936206818 (0.22814841934463434) + | > loss_disc_real_2: 0.2313259392976761 (0.23590304475953008) + | > loss_disc_real_3: 0.2331075519323349 (0.23625572413554666) + | > loss_disc_real_4: 0.2236078381538391 (0.2354268926190354) + | > loss_disc_real_5: 0.21861177682876587 (0.23821260993592222) + | > loss_0: 2.7821238040924072 (2.74363538675141) + | > grad_norm_0: tensor(12.6644, device='cuda:0') (tensor(11.4207, device='cuda:0')) + | > loss_gen: 1.8695203065872192 (1.8734985422669796) + | > loss_kl: 2.4352104663848877 (2.488872854333177) + | > loss_feat: 2.9153122901916504 (3.000506626235114) + | > loss_mel: 22.808549880981445 (22.86352496119271) + | > loss_duration: 1.428657054901123 (1.401518358124626) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.457250595092773 (31.627921305204694) + | > grad_norm_1: tensor(176.0545, device='cuda:0') (tensor(127.9493, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.9338 (0.8622156402520964) + | > loader_time: 0.0179 (0.012646230340701102) + + + --> TIME: 2024-06-25 05:02:03 -- STEP: 367/591 -- GLOBAL_STEP: 8050 + | > loss_disc: 2.7063231468200684 (2.7431655566763826) + | > loss_disc_real_0: 0.27678799629211426 (0.22015084005506552) + | > loss_disc_real_1: 0.23966160416603088 (0.22816962868543672) + | > loss_disc_real_2: 0.22391685843467712 (0.23601964302868544) + | > loss_disc_real_3: 0.2740357220172882 (0.23607429589984852) + | > loss_disc_real_4: 0.21343211829662323 (0.23536217427870884) + | > loss_disc_real_5: 0.2259984314441681 (0.23814359762038456) + | > loss_0: 2.7063231468200684 (2.7431655566763826) + | > grad_norm_0: tensor(15.1233, device='cuda:0') (tensor(11.4730, device='cuda:0')) + | > loss_gen: 1.8853763341903687 (1.8739940577696714) + | > loss_kl: 2.530470132827759 (2.4869027819880367) + | > loss_feat: 3.090745449066162 (2.99857609408428) + | > loss_mel: 22.948055267333984 (22.85330720948264) + | > loss_duration: 1.4341495037078857 (1.4032519808906938) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.888795852661133 (31.61603208848501) + | > grad_norm_1: tensor(128.0454, device='cuda:0') (tensor(128.7254, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.9775 (0.8678258355371957) + | > loader_time: 0.0175 (0.012897993628270625) + + + --> TIME: 2024-06-25 05:02:28 -- STEP: 392/591 -- GLOBAL_STEP: 8075 + | > loss_disc: 2.8009324073791504 (2.7451074524801617) + | > loss_disc_real_0: 0.1507759988307953 (0.21987886700247014) + | > loss_disc_real_1: 0.2657318115234375 (0.22864677027171973) + | > loss_disc_real_2: 0.2298336923122406 (0.2363820661011399) + | > loss_disc_real_3: 0.27576935291290283 (0.23647511849293903) + | > loss_disc_real_4: 0.23943310976028442 (0.23558532305974134) + | > loss_disc_real_5: 0.2349701225757599 (0.23818671665325455) + | > loss_0: 2.8009324073791504 (2.7451074524801617) + | > grad_norm_0: tensor(16.8312, device='cuda:0') (tensor(11.5851, device='cuda:0')) + | > loss_gen: 1.8272804021835327 (1.873316373752088) + | > loss_kl: 2.5128414630889893 (2.4845245219006835) + | > loss_feat: 2.8119640350341797 (2.991638638535324) + | > loss_mel: 22.467308044433594 (22.8421197959355) + | > loss_duration: 1.4381654262542725 (1.4041939882599566) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.057559967041016 (31.595793276417012) + | > grad_norm_1: tensor(184.7551, device='cuda:0') (tensor(130.2586, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.9967 (0.8742739321017751) + | > loader_time: 0.0175 (0.0131600845833214) + + + --> TIME: 2024-06-25 05:02:53 -- STEP: 417/591 -- GLOBAL_STEP: 8100 + | > loss_disc: 2.7473702430725098 (2.746310933888388) + | > loss_disc_real_0: 0.15666139125823975 (0.2199522856471064) + | > loss_disc_real_1: 0.21824151277542114 (0.2288371799804038) + | > loss_disc_real_2: 0.2111317217350006 (0.23632806335850584) + | > loss_disc_real_3: 0.2664065957069397 (0.23643083877557758) + | > loss_disc_real_4: 0.21784520149230957 (0.23558802699013578) + | > loss_disc_real_5: 0.20333240926265717 (0.23820120492856278) + | > loss_0: 2.7473702430725098 (2.746310933888388) + | > grad_norm_0: tensor(12.7419, device='cuda:0') (tensor(11.6054, device='cuda:0')) + | > loss_gen: 1.917763352394104 (1.8710135307243403) + | > loss_kl: 2.408616542816162 (2.481372334116654) + | > loss_feat: 2.7014529705047607 (2.9805717765570257) + | > loss_mel: 22.070613861083984 (22.814273600955666) + | > loss_duration: 1.4126076698303223 (1.4050012272324768) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.51105308532715 (31.552232431279098) + | > grad_norm_1: tensor(78.8608, device='cuda:0') (tensor(129.3161, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 0.9901 (0.880870319956498) + | > loader_time: 0.0192 (0.013466274709724411) + + + --> TIME: 2024-06-25 05:03:19 -- STEP: 442/591 -- GLOBAL_STEP: 8125 + | > loss_disc: 2.7561466693878174 (2.7466158057769507) + | > loss_disc_real_0: 0.3129623234272003 (0.2199745002577747) + | > loss_disc_real_1: 0.2749566435813904 (0.22905572992644158) + | > loss_disc_real_2: 0.2474621832370758 (0.23643739076491394) + | > loss_disc_real_3: 0.2198764532804489 (0.2364135614819656) + | > loss_disc_real_4: 0.25626686215400696 (0.23556994498450293) + | > loss_disc_real_5: 0.22389854490756989 (0.23808344106328974) + | > loss_0: 2.7561466693878174 (2.7466158057769507) + | > grad_norm_0: tensor(14.2331, device='cuda:0') (tensor(11.7261, device='cuda:0')) + | > loss_gen: 1.892622947692871 (1.8701327944772814) + | > loss_kl: 2.445615530014038 (2.4796619728140157) + | > loss_feat: 2.734409809112549 (2.9715223781663367) + | > loss_mel: 22.34217071533203 (22.79626371633953) + | > loss_duration: 1.3996511697769165 (1.4056561311445614) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.814468383789062 (31.523236978107988) + | > grad_norm_1: tensor(63.7409, device='cuda:0') (tensor(129.2687, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 1.0081 (0.887927933516006) + | > loader_time: 0.0204 (0.013771154761853805) + + + --> TIME: 2024-06-25 05:03:45 -- STEP: 467/591 -- GLOBAL_STEP: 8150 + | > loss_disc: 2.7263426780700684 (2.7466328863924026) + | > loss_disc_real_0: 0.18222154676914215 (0.2200734801032048) + | > loss_disc_real_1: 0.22140227258205414 (0.2289730513695731) + | > loss_disc_real_2: 0.2500510513782501 (0.23644781390272832) + | > loss_disc_real_3: 0.23382192850112915 (0.2363880799018341) + | > loss_disc_real_4: 0.25229161977767944 (0.23558314849784084) + | > loss_disc_real_5: 0.2536020278930664 (0.2381105859797823) + | > loss_0: 2.7263426780700684 (2.7466328863924026) + | > grad_norm_0: tensor(8.6436, device='cuda:0') (tensor(11.6687, device='cuda:0')) + | > loss_gen: 1.8070521354675293 (1.8691239119597114) + | > loss_kl: 2.443650722503662 (2.4773746562974095) + | > loss_feat: 2.8079118728637695 (2.966052662892311) + | > loss_mel: 22.250934600830078 (22.764553670454337) + | > loss_duration: 1.407430648803711 (1.4068614001958213) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.71697998046875 (31.48396626784919) + | > grad_norm_1: tensor(65.9419, device='cuda:0') (tensor(129.3309, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 1.071 (0.8950614873040665) + | > loader_time: 0.0175 (0.014029743329360606) + + + --> TIME: 2024-06-25 05:04:11 -- STEP: 492/591 -- GLOBAL_STEP: 8175 + | > loss_disc: 2.734830856323242 (2.74650738055144) + | > loss_disc_real_0: 0.2332567274570465 (0.2200836180520009) + | > loss_disc_real_1: 0.2776692509651184 (0.2289579898058399) + | > loss_disc_real_2: 0.28357136249542236 (0.23649045064803062) + | > loss_disc_real_3: 0.258229523897171 (0.23637161267603315) + | > loss_disc_real_4: 0.2264752835035324 (0.23553872120574237) + | > loss_disc_real_5: 0.25330159068107605 (0.23804155073878241) + | > loss_0: 2.734830856323242 (2.74650738055144) + | > grad_norm_0: tensor(6.6443, device='cuda:0') (tensor(11.6385, device='cuda:0')) + | > loss_gen: 1.6620949506759644 (1.868293896680925) + | > loss_kl: 2.5618369579315186 (2.4755886158322897) + | > loss_feat: 2.7589800357818604 (2.9609946288713593) + | > loss_mel: 22.48621940612793 (22.745925306304688) + | > loss_duration: 1.4382790327072144 (1.4081900689175448) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.90740966796875 (31.458992485108414) + | > grad_norm_1: tensor(68.2980, device='cuda:0') (tensor(128.0281, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 1.0518 (0.902660945566689) + | > loader_time: 0.0203 (0.01433101466031579) + + + --> TIME: 2024-06-25 05:04:40 -- STEP: 517/591 -- GLOBAL_STEP: 8200 + | > loss_disc: 2.770724296569824 (2.7465784056274303) + | > loss_disc_real_0: 0.3108956813812256 (0.22006752202197247) + | > loss_disc_real_1: 0.22552455961704254 (0.2290339377101662) + | > loss_disc_real_2: 0.18807615339756012 (0.23644488974973357) + | > loss_disc_real_3: 0.21738900244235992 (0.23627456152231596) + | > loss_disc_real_4: 0.25811970233917236 (0.23564299306970962) + | > loss_disc_real_5: 0.24844415485858917 (0.23794546510080306) + | > loss_0: 2.770724296569824 (2.7465784056274303) + | > grad_norm_0: tensor(18.8872, device='cuda:0') (tensor(11.5952, device='cuda:0')) + | > loss_gen: 1.7122782468795776 (1.8681530306832703) + | > loss_kl: 2.4053094387054443 (2.4734151755356923) + | > loss_feat: 3.0268759727478027 (2.9578798508966915) + | > loss_mel: 22.72390365600586 (22.729395366499) + | > loss_duration: 1.4519670009613037 (1.4095858623718607) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.320335388183594 (31.438429255088938) + | > grad_norm_1: tensor(77.7206, device='cuda:0') (tensor(126.6417, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 1.1053 (0.9121136517773744) + | > loader_time: 0.021 (0.01464567700944971) + + + --> TIME: 2024-06-25 05:05:09 -- STEP: 542/591 -- GLOBAL_STEP: 8225 + | > loss_disc: 2.7111546993255615 (2.7463396695267237) + | > loss_disc_real_0: 0.29065459966659546 (0.21999632110784853) + | > loss_disc_real_1: 0.24204832315444946 (0.22906697810905885) + | > loss_disc_real_2: 0.2130068689584732 (0.23645190407979091) + | > loss_disc_real_3: 0.2523816227912903 (0.23622408206832365) + | > loss_disc_real_4: 0.23590168356895447 (0.2355468881306173) + | > loss_disc_real_5: 0.2496797889471054 (0.23796679038293247) + | > loss_0: 2.7111546993255615 (2.7463396695267237) + | > grad_norm_0: tensor(12.9530, device='cuda:0') (tensor(11.6754, device='cuda:0')) + | > loss_gen: 1.7031762599945068 (1.8674128660416691) + | > loss_kl: 2.495333433151245 (2.471219754307032) + | > loss_feat: 2.9226667881011963 (2.951742106698096) + | > loss_mel: 23.58396339416504 (22.714787096115064) + | > loss_duration: 1.4832141399383545 (1.410962441530614) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.1883544921875 (31.41612423126108) + | > grad_norm_1: tensor(66.8207, device='cuda:0') (tensor(125.3871, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 1.1419 (0.9234463922212045) + | > loader_time: 0.0236 (0.014960281083504655) + + + --> TIME: 2024-06-25 05:05:41 -- STEP: 567/591 -- GLOBAL_STEP: 8250 + | > loss_disc: 2.728567600250244 (2.7465082026453045) + | > loss_disc_real_0: 0.21265047788619995 (0.21987049105516393) + | > loss_disc_real_1: 0.23602525889873505 (0.228984623175029) + | > loss_disc_real_2: 0.23087218403816223 (0.23652836346647307) + | > loss_disc_real_3: 0.2187754362821579 (0.23629908878651876) + | > loss_disc_real_4: 0.2086559236049652 (0.235581204689369) + | > loss_disc_real_5: 0.2510012090206146 (0.23800711005242833) + | > loss_0: 2.728567600250244 (2.7465082026453045) + | > grad_norm_0: tensor(3.6897, device='cuda:0') (tensor(11.6605, device='cuda:0')) + | > loss_gen: 1.8138858079910278 (1.867321975647457) + | > loss_kl: 2.4687047004699707 (2.4690835909027684) + | > loss_feat: 2.877465009689331 (2.946853776036957) + | > loss_mel: 22.92721176147461 (22.697450758918894) + | > loss_duration: 1.466184139251709 (1.4125247739610207) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.553451538085938 (31.393234824797883) + | > grad_norm_1: tensor(72.9087, device='cuda:0') (tensor(126.5520, device='cuda:0')) + | > current_lr_0: 0.00019967524363831608 + | > current_lr_1: 0.00019967524363831608 + | > step_time: 1.3483 (0.938115320087951) + | > loader_time: 0.0264 (0.015316772292740975) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037360749346144655 (+0.0001656552578540556) + | > avg_loss_disc: 2.7499150113856543 (+0.009167711785499577) + | > avg_loss_disc_real_0: 0.17832856609466227 (-0.022478577304393704) + | > avg_loss_disc_real_1: 0.13857104280527605 (-0.07494150752082782) + | > avg_loss_disc_real_2: 0.18817077006431335 (-0.035449892916577935) + | > avg_loss_disc_real_3: 0.2481809261631458 (+0.049908330148838886) + | > avg_loss_disc_real_4: 0.21139277454386365 (-0.009025866998002896) + | > avg_loss_disc_real_5: 0.24006439618607786 (+0.0031287127986867613) + | > avg_loss_0: 2.7499150113856543 (+0.009167711785499577) + | > avg_loss_gen: 1.596160979981118 (-0.10545374484772374) + | > avg_loss_kl: 2.5581344188527857 (+0.07326581630301021) + | > avg_loss_feat: 2.9382391990499293 (+0.08947057419634863) + | > avg_loss_mel: 22.619835670958174 (+0.22714233398438566) + | > avg_loss_duration: 1.401861500232778 (+0.0001337578956119856) + | > avg_loss_1: 31.114231718347426 (+0.2845588440590703) + + + > EPOCH: 15/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 05:06:20)  + + --> TIME: 2024-06-25 05:06:22 -- STEP: 1/591 -- GLOBAL_STEP: 8275 + | > loss_disc: 2.7178475856781006 (2.7178475856781006) + | > loss_disc_real_0: 0.2926176190376282 (0.2926176190376282) + | > loss_disc_real_1: 0.23399442434310913 (0.23399442434310913) + | > loss_disc_real_2: 0.26531004905700684 (0.26531004905700684) + | > loss_disc_real_3: 0.21102894842624664 (0.21102894842624664) + | > loss_disc_real_4: 0.20646898448467255 (0.20646898448467255) + | > loss_disc_real_5: 0.20933444797992706 (0.20933444797992706) + | > loss_0: 2.7178475856781006 (2.7178475856781006) + | > grad_norm_0: tensor(13.2806, device='cuda:0') (tensor(13.2806, device='cuda:0')) + | > loss_gen: 2.0748648643493652 (2.0748648643493652) + | > loss_kl: 2.7967684268951416 (2.7967684268951416) + | > loss_feat: 3.268545627593994 (3.268545627593994) + | > loss_mel: 22.840595245361328 (22.840595245361328) + | > loss_duration: 1.4245922565460205 (1.4245922565460205) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.40536880493164 (32.40536880493164) + | > grad_norm_1: tensor(260.2139, device='cuda:0') (tensor(260.2139, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.7523 (0.7522892951965332) + | > loader_time: 0.0071 (0.007093667984008789) + + + --> TIME: 2024-06-25 05:06:42 -- STEP: 26/591 -- GLOBAL_STEP: 8300 + | > loss_disc: 2.8809573650360107 (2.838285492016719) + | > loss_disc_real_0: 0.3486817479133606 (0.21874007124167222) + | > loss_disc_real_1: 0.21924801170825958 (0.27895208505483776) + | > loss_disc_real_2: 0.26058125495910645 (0.2515706677849475) + | > loss_disc_real_3: 0.28126922249794006 (0.23792781508885896) + | > loss_disc_real_4: 0.24021022021770477 (0.24344154791190073) + | > loss_disc_real_5: 0.27112334966659546 (0.23805390699551657) + | > loss_0: 2.8809573650360107 (2.838285492016719) + | > grad_norm_0: tensor(20.7892, device='cuda:0') (tensor(10.3103, device='cuda:0')) + | > loss_gen: 1.594825029373169 (1.9710940902049727) + | > loss_kl: 2.624537706375122 (2.660579048670255) + | > loss_feat: 2.9084794521331787 (3.1768260460633497) + | > loss_mel: 22.834653854370117 (23.106494169968826) + | > loss_duration: 1.3253366947174072 (1.3655072175539458) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.28783416748047 (32.28050085214467) + | > grad_norm_1: tensor(146.2356, device='cuda:0') (tensor(178.8905, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8045 (0.7771608646099384) + | > loader_time: 0.0093 (0.008814380719111517) + + + --> TIME: 2024-06-25 05:07:02 -- STEP: 51/591 -- GLOBAL_STEP: 8325 + | > loss_disc: 2.843040943145752 (2.841861224641987) + | > loss_disc_real_0: 0.1295827031135559 (0.21812109561527476) + | > loss_disc_real_1: 0.24476949870586395 (0.2800751424887601) + | > loss_disc_real_2: 0.2529286742210388 (0.24820505550094674) + | > loss_disc_real_3: 0.22058571875095367 (0.2363681235149795) + | > loss_disc_real_4: 0.23364105820655823 (0.24095497265750287) + | > loss_disc_real_5: 0.24604953825473785 (0.23783889617405685) + | > loss_0: 2.843040943145752 (2.841861224641987) + | > grad_norm_0: tensor(20.0657, device='cuda:0') (tensor(11.5838, device='cuda:0')) + | > loss_gen: 1.977834939956665 (1.9039658728767843) + | > loss_kl: 2.5222327709198 (2.5824569580601717) + | > loss_feat: 2.8960371017456055 (2.9546594479504753) + | > loss_mel: 22.43226432800293 (22.972999535354912) + | > loss_duration: 1.3811426162719727 (1.381826078190523) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.209510803222656 (31.79590804904114) + | > grad_norm_1: tensor(79.0900, device='cuda:0') (tensor(147.7948, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8014 (0.7888648930717918) + | > loader_time: 0.0097 (0.009266862682267733) + + + --> TIME: 2024-06-25 05:07:23 -- STEP: 76/591 -- GLOBAL_STEP: 8350 + | > loss_disc: 2.7824196815490723 (2.8169680331882683) + | > loss_disc_real_0: 0.12218984961509705 (0.21723560086990656) + | > loss_disc_real_1: 0.24435830116271973 (0.2691477958701159) + | > loss_disc_real_2: 0.224114328622818 (0.24357808695027697) + | > loss_disc_real_3: 0.2381220906972885 (0.23602081286279777) + | > loss_disc_real_4: 0.24429666996002197 (0.23927801474928856) + | > loss_disc_real_5: 0.21539296209812164 (0.23722586682752558) + | > loss_0: 2.7824196815490723 (2.8169680331882683) + | > grad_norm_0: tensor(21.8721, device='cuda:0') (tensor(10.7008, device='cuda:0')) + | > loss_gen: 1.9090592861175537 (1.8749089993928607) + | > loss_kl: 2.41475772857666 (2.5550312368493344) + | > loss_feat: 2.9265940189361572 (2.9045797115878056) + | > loss_mel: 22.926485061645508 (22.807038809123792) + | > loss_duration: 1.3843984603881836 (1.3883382552548458) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.561294555664062 (31.529897012208632) + | > grad_norm_1: tensor(103.3523, device='cuda:0') (tensor(145.5961, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8104 (0.7982931450793617) + | > loader_time: 0.0107 (0.009634664184168765) + + + --> TIME: 2024-06-25 05:07:44 -- STEP: 101/591 -- GLOBAL_STEP: 8375 + | > loss_disc: 2.7055697441101074 (2.806551326619517) + | > loss_disc_real_0: 0.16818572580814362 (0.21851538320874223) + | > loss_disc_real_1: 0.2451462596654892 (0.2625490955196985) + | > loss_disc_real_2: 0.23817376792430878 (0.24064187233400813) + | > loss_disc_real_3: 0.270820677280426 (0.2360659296264743) + | > loss_disc_real_4: 0.22235842049121857 (0.2379765346793845) + | > loss_disc_real_5: 0.22330524027347565 (0.23743705112155122) + | > loss_0: 2.7055697441101074 (2.806551326619517) + | > grad_norm_0: tensor(11.7629, device='cuda:0') (tensor(11.0883, device='cuda:0')) + | > loss_gen: 1.934641718864441 (1.861970884965198) + | > loss_kl: 2.539393424987793 (2.548697655743894) + | > loss_feat: 2.9422998428344727 (2.890208914728448) + | > loss_mel: 21.980304718017578 (22.727431703322004) + | > loss_duration: 1.4089384078979492 (1.3965380569495773) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.80557632446289 (31.424847140170556) + | > grad_norm_1: tensor(64.1936, device='cuda:0') (tensor(139.0226, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8383 (0.8063451441207735) + | > loader_time: 0.0121 (0.009988409457820476) + + + --> TIME: 2024-06-25 05:08:06 -- STEP: 126/591 -- GLOBAL_STEP: 8400 + | > loss_disc: 2.7437562942504883 (2.7954735055802367) + | > loss_disc_real_0: 0.24719595909118652 (0.2186442900389906) + | > loss_disc_real_1: 0.2292216569185257 (0.2569141678866886) + | > loss_disc_real_2: 0.23443686962127686 (0.23930470302464468) + | > loss_disc_real_3: 0.23091983795166016 (0.23571268256221498) + | > loss_disc_real_4: 0.2597578167915344 (0.23776998264448984) + | > loss_disc_real_5: 0.27527424693107605 (0.23794025005329222) + | > loss_0: 2.7437562942504883 (2.7954735055802367) + | > grad_norm_0: tensor(7.6611, device='cuda:0') (tensor(10.8963, device='cuda:0')) + | > loss_gen: 1.7525469064712524 (1.8581604408839392) + | > loss_kl: 2.645069122314453 (2.538871471844024) + | > loss_feat: 3.0639150142669678 (2.9017211425872076) + | > loss_mel: 22.92714500427246 (22.703263176812065) + | > loss_duration: 1.399863839149475 (1.3981962497272187) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.78853988647461 (31.40021246955508) + | > grad_norm_1: tensor(141.7691, device='cuda:0') (tensor(135.0333, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8611 (0.8133689619245983) + | > loader_time: 0.0122 (0.010248197449578179) + + + --> TIME: 2024-06-25 05:08:27 -- STEP: 151/591 -- GLOBAL_STEP: 8425 + | > loss_disc: 2.717766046524048 (2.784592713741278) + | > loss_disc_real_0: 0.2374703586101532 (0.2187480620595793) + | > loss_disc_real_1: 0.23836438357830048 (0.2528906940229681) + | > loss_disc_real_2: 0.22995412349700928 (0.23818736853978492) + | > loss_disc_real_3: 0.2515016794204712 (0.23558160445548051) + | > loss_disc_real_4: 0.26298341155052185 (0.23704243406949455) + | > loss_disc_real_5: 0.24444755911827087 (0.23772570431627185) + | > loss_0: 2.717766046524048 (2.784592713741278) + | > grad_norm_0: tensor(5.9355, device='cuda:0') (tensor(11.1377, device='cuda:0')) + | > loss_gen: 1.7070622444152832 (1.8618457001566098) + | > loss_kl: 2.4672765731811523 (2.5285731988237408) + | > loss_feat: 3.0703396797180176 (2.924466651007039) + | > loss_mel: 22.487722396850586 (22.7040645397262) + | > loss_duration: 1.3722844123840332 (1.3967947509904568) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.104684829711914 (31.415744857282828) + | > grad_norm_1: tensor(85.0746, device='cuda:0') (tensor(136.4821, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8577 (0.8188181065565702) + | > loader_time: 0.0113 (0.010408804116659608) + + + --> TIME: 2024-06-25 05:08:49 -- STEP: 176/591 -- GLOBAL_STEP: 8450 + | > loss_disc: 2.736257553100586 (2.7783583768389444) + | > loss_disc_real_0: 0.14411196112632751 (0.21842532816597007) + | > loss_disc_real_1: 0.16282223165035248 (0.2493059627542441) + | > loss_disc_real_2: 0.22428551316261292 (0.23799290194768796) + | > loss_disc_real_3: 0.23111394047737122 (0.23558001287958838) + | > loss_disc_real_4: 0.18538528680801392 (0.23681134239516474) + | > loss_disc_real_5: 0.1847856342792511 (0.2375661303204569) + | > loss_0: 2.736257553100586 (2.7783583768389444) + | > grad_norm_0: tensor(17.6707, device='cuda:0') (tensor(11.3067, device='cuda:0')) + | > loss_gen: 2.029996395111084 (1.8663891391320662) + | > loss_kl: 2.505037546157837 (2.5258001603863467) + | > loss_feat: 3.0195536613464355 (2.9423442848704067) + | > loss_mel: 22.251449584960938 (22.695803132924162) + | > loss_duration: 1.3635787963867188 (1.3946966041218145) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.16961669921875 (31.425033341754567) + | > grad_norm_1: tensor(158.7111, device='cuda:0') (tensor(130.7674, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8928 (0.8247270882129669) + | > loader_time: 0.0124 (0.01063998585397547) + + + --> TIME: 2024-06-25 05:09:11 -- STEP: 201/591 -- GLOBAL_STEP: 8475 + | > loss_disc: 2.773214101791382 (2.775505883183646) + | > loss_disc_real_0: 0.31544291973114014 (0.21973362068335214) + | > loss_disc_real_1: 0.23660634458065033 (0.24711490515156168) + | > loss_disc_real_2: 0.21783018112182617 (0.23751339442397826) + | > loss_disc_real_3: 0.27371442317962646 (0.2358006385724936) + | > loss_disc_real_4: 0.2107570320367813 (0.23660812367550768) + | > loss_disc_real_5: 0.26020684838294983 (0.23786251736221029) + | > loss_0: 2.773214101791382 (2.775505883183646) + | > grad_norm_0: tensor(17.5394, device='cuda:0') (tensor(11.5891, device='cuda:0')) + | > loss_gen: 1.7231836318969727 (1.8671406892994744) + | > loss_kl: 2.571260690689087 (2.5234344325848492) + | > loss_feat: 2.9294683933258057 (2.9519515120567954) + | > loss_mel: 22.732629776000977 (22.691973007733544) + | > loss_duration: 1.361490249633789 (1.3941887961098205) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.31803321838379 (31.42868846684546) + | > grad_norm_1: tensor(116.2652, device='cuda:0') (tensor(130.0345, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8732 (0.830260811753534) + | > loader_time: 0.0118 (0.010872919168045272) + + + --> TIME: 2024-06-25 05:09:34 -- STEP: 226/591 -- GLOBAL_STEP: 8500 + | > loss_disc: 2.7235798835754395 (2.771486109336921) + | > loss_disc_real_0: 0.1330002397298813 (0.21934221506909987) + | > loss_disc_real_1: 0.22994926571846008 (0.24483007416788452) + | > loss_disc_real_2: 0.24181033670902252 (0.23754225743819127) + | > loss_disc_real_3: 0.19440504908561707 (0.23538530189379125) + | > loss_disc_real_4: 0.25593337416648865 (0.2365000532791678) + | > loss_disc_real_5: 0.2327423095703125 (0.2378256723812196) + | > loss_0: 2.7235798835754395 (2.771486109336921) + | > grad_norm_0: tensor(16.7072, device='cuda:0') (tensor(11.6809, device='cuda:0')) + | > loss_gen: 1.869053602218628 (1.867067455190473) + | > loss_kl: 2.526230573654175 (2.5209007242084613) + | > loss_feat: 3.0185117721557617 (2.954449606152761) + | > loss_mel: 22.697795867919922 (22.67971245588454) + | > loss_duration: 1.3674423694610596 (1.3935143013971036) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.479034423828125 (31.41564456129496) + | > grad_norm_1: tensor(36.8336, device='cuda:0') (tensor(129.4242, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8889 (0.8353774790215281) + | > loader_time: 0.0152 (0.011194544555866612) + + + --> TIME: 2024-06-25 05:09:57 -- STEP: 251/591 -- GLOBAL_STEP: 8525 + | > loss_disc: 3.003952741622925 (2.768448457299951) + | > loss_disc_real_0: 0.23099324107170105 (0.21948127323887737) + | > loss_disc_real_1: 0.2580265402793884 (0.24328126464469496) + | > loss_disc_real_2: 0.08659180998802185 (0.23680323967895658) + | > loss_disc_real_3: 0.2141856998205185 (0.2354688273958001) + | > loss_disc_real_4: 0.2669772803783417 (0.2362928712273024) + | > loss_disc_real_5: 0.24482010304927826 (0.23774834036589618) + | > loss_0: 3.003952741622925 (2.768448457299951) + | > grad_norm_0: tensor(28.6027, device='cuda:0') (tensor(11.6508, device='cuda:0')) + | > loss_gen: 2.162412166595459 (1.8695572728654777) + | > loss_kl: 2.5217909812927246 (2.5195492605764085) + | > loss_feat: 2.9785468578338623 (2.963780420234953) + | > loss_mel: 22.6962947845459 (22.679171793964276) + | > loss_duration: 1.3978074789047241 (1.394120499432324) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.756853103637695 (31.426179277944374) + | > grad_norm_1: tensor(206.3666, device='cuda:0') (tensor(132.0003, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8863 (0.8407760549826451) + | > loader_time: 0.0133 (0.011496792751479433) + + + --> TIME: 2024-06-25 05:10:19 -- STEP: 276/591 -- GLOBAL_STEP: 8550 + | > loss_disc: 2.729173183441162 (2.766640204450359) + | > loss_disc_real_0: 0.16142067313194275 (0.21920389320323433) + | > loss_disc_real_1: 0.17378634214401245 (0.24193549463930333) + | > loss_disc_real_2: 0.27672311663627625 (0.23797754597836646) + | > loss_disc_real_3: 0.22838586568832397 (0.23522733778193378) + | > loss_disc_real_4: 0.2506978511810303 (0.23628287935170575) + | > loss_disc_real_5: 0.26036015152931213 (0.23781044856793637) + | > loss_0: 2.729173183441162 (2.766640204450359) + | > grad_norm_0: tensor(13.9466, device='cuda:0') (tensor(11.8231, device='cuda:0')) + | > loss_gen: 1.9550107717514038 (1.8683100152706755) + | > loss_kl: 2.479356288909912 (2.5180609088013144) + | > loss_feat: 2.9871232509613037 (2.961148290530494) + | > loss_mel: 22.940793991088867 (22.66862856823465) + | > loss_duration: 1.400651216506958 (1.3941907092281005) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.762935638427734 (31.410338519276053) + | > grad_norm_1: tensor(67.3795, device='cuda:0') (tensor(130.9641, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.8922 (0.8459324810815894) + | > loader_time: 0.0167 (0.011797812537870545) + + + --> TIME: 2024-06-25 05:10:43 -- STEP: 301/591 -- GLOBAL_STEP: 8575 + | > loss_disc: 2.757779121398926 (2.76534429182642) + | > loss_disc_real_0: 0.3050283193588257 (0.21952780927732532) + | > loss_disc_real_1: 0.18944764137268066 (0.24104173407206103) + | > loss_disc_real_2: 0.2507060468196869 (0.2379705611356469) + | > loss_disc_real_3: 0.22654466331005096 (0.23524815121362375) + | > loss_disc_real_4: 0.2705453932285309 (0.2361314768014952) + | > loss_disc_real_5: 0.2465771734714508 (0.23779255592347776) + | > loss_0: 2.757779121398926 (2.76534429182642) + | > grad_norm_0: tensor(18.0928, device='cuda:0') (tensor(11.8992, device='cuda:0')) + | > loss_gen: 1.6965643167495728 (1.867560460321927) + | > loss_kl: 2.611619710922241 (2.5142181553317866) + | > loss_feat: 2.832277536392212 (2.9612282162092844) + | > loss_mel: 21.936983108520508 (22.64681904814964) + | > loss_duration: 1.4065368175506592 (1.3945943148825253) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.483980178833008 (31.384420185786148) + | > grad_norm_1: tensor(43.1065, device='cuda:0') (tensor(129.2014, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.9028 (0.8521278664915268) + | > loader_time: 0.0158 (0.012058628753966266) + + + --> TIME: 2024-06-25 05:11:07 -- STEP: 326/591 -- GLOBAL_STEP: 8600 + | > loss_disc: 2.718085527420044 (2.763285431393818) + | > loss_disc_real_0: 0.21993228793144226 (0.2193116009509636) + | > loss_disc_real_1: 0.2121569812297821 (0.23979413948176093) + | > loss_disc_real_2: 0.2409452199935913 (0.23775168416873077) + | > loss_disc_real_3: 0.24673865735530853 (0.23539791653858372) + | > loss_disc_real_4: 0.223255917429924 (0.23585170155471089) + | > loss_disc_real_5: 0.19312192499637604 (0.2377016643987843) + | > loss_0: 2.718085527420044 (2.763285431393818) + | > grad_norm_0: tensor(8.1198, device='cuda:0') (tensor(11.9872, device='cuda:0')) + | > loss_gen: 2.0082924365997314 (1.8677992433126718) + | > loss_kl: 2.4141910076141357 (2.51222213060578) + | > loss_feat: 3.0092291831970215 (2.963931854517183) + | > loss_mel: 22.66151237487793 (22.64138210039197) + | > loss_duration: 1.412714958190918 (1.3952944220209411) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.505939483642578 (31.380629779371016) + | > grad_norm_1: tensor(142.2035, device='cuda:0') (tensor(129.4385, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.9251 (0.8575811722527252) + | > loader_time: 0.0139 (0.012283224269656315) + + + --> TIME: 2024-06-25 05:11:31 -- STEP: 351/591 -- GLOBAL_STEP: 8625 + | > loss_disc: 2.9379286766052246 (2.763500301926225) + | > loss_disc_real_0: 0.3255694508552551 (0.2196448904241931) + | > loss_disc_real_1: 0.29609474539756775 (0.23974404863991963) + | > loss_disc_real_2: 0.28968480229377747 (0.23785134309377426) + | > loss_disc_real_3: 0.27269139885902405 (0.2355260150215225) + | > loss_disc_real_4: 0.2368449717760086 (0.2358978501497171) + | > loss_disc_real_5: 0.2540307343006134 (0.23777413363979752) + | > loss_0: 2.9379286766052246 (2.763500301926225) + | > grad_norm_0: tensor(24.3707, device='cuda:0') (tensor(12.0670, device='cuda:0')) + | > loss_gen: 1.7149343490600586 (1.868530723783705) + | > loss_kl: 2.583495616912842 (2.5115959311482583) + | > loss_feat: 2.769690990447998 (2.9634969655604797) + | > loss_mel: 22.39046287536621 (22.631739146349428) + | > loss_duration: 1.3624167442321777 (1.3963903884942024) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.821001052856445 (31.371753165524908) + | > grad_norm_1: tensor(120.4370, device='cuda:0') (tensor(132.3228, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.9995 (0.863448580445727) + | > loader_time: 0.0159 (0.012543161370475748) + + + --> TIME: 2024-06-25 05:11:55 -- STEP: 376/591 -- GLOBAL_STEP: 8650 + | > loss_disc: 2.768449544906616 (2.7662709139763053) + | > loss_disc_real_0: 0.1548181027173996 (0.2195534551952113) + | > loss_disc_real_1: 0.275277704000473 (0.24024325918326983) + | > loss_disc_real_2: 0.22063031792640686 (0.23805896224493675) + | > loss_disc_real_3: 0.21595151722431183 (0.23567718980794258) + | > loss_disc_real_4: 0.23757252097129822 (0.23622912231912005) + | > loss_disc_real_5: 0.25722554326057434 (0.2378218937665224) + | > loss_0: 2.768449544906616 (2.7662709139763053) + | > grad_norm_0: tensor(14.6036, device='cuda:0') (tensor(12.2306, device='cuda:0')) + | > loss_gen: 1.7303739786148071 (1.866707886470125) + | > loss_kl: 2.5260093212127686 (2.5101511934970295) + | > loss_feat: 2.9812920093536377 (2.9508326852575273) + | > loss_mel: 22.895427703857422 (22.622539707954893) + | > loss_duration: 1.4341623783111572 (1.3979722220212838) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.567264556884766 (31.348203709784976) + | > grad_norm_1: tensor(188.2628, device='cuda:0') (tensor(133.0797, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.9405 (0.8697757308787488) + | > loader_time: 0.0172 (0.012799474787204824) + + + --> TIME: 2024-06-25 05:12:20 -- STEP: 401/591 -- GLOBAL_STEP: 8675 + | > loss_disc: 2.752134084701538 (2.7651627735604087) + | > loss_disc_real_0: 0.2897549867630005 (0.2196883230732563) + | > loss_disc_real_1: 0.2677897810935974 (0.23977823206165483) + | > loss_disc_real_2: 0.24594590067863464 (0.23800967804660225) + | > loss_disc_real_3: 0.2379586398601532 (0.23551625065375445) + | > loss_disc_real_4: 0.24397920072078705 (0.23618545526280962) + | > loss_disc_real_5: 0.25766295194625854 (0.23795866821323547) + | > loss_0: 2.752134084701538 (2.7651627735604087) + | > grad_norm_0: tensor(14.8798, device='cuda:0') (tensor(12.1691, device='cuda:0')) + | > loss_gen: 1.738550066947937 (1.8659046353247397) + | > loss_kl: 2.437887191772461 (2.508525970868043) + | > loss_feat: 2.7921953201293945 (2.9463923542279566) + | > loss_mel: 22.000612258911133 (22.606912532054874) + | > loss_duration: 1.4101386070251465 (1.3987841956930547) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.379383087158203 (31.326519687871386) + | > grad_norm_1: tensor(75.3441, device='cuda:0') (tensor(131.0527, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.9948 (0.8765701700624385) + | > loader_time: 0.0158 (0.013079473205338095) + + + --> TIME: 2024-06-25 05:12:45 -- STEP: 426/591 -- GLOBAL_STEP: 8700 + | > loss_disc: 2.6963703632354736 (2.7637312692095986) + | > loss_disc_real_0: 0.19078093767166138 (0.2192897129226738) + | > loss_disc_real_1: 0.25013667345046997 (0.2390045677901993) + | > loss_disc_real_2: 0.24652427434921265 (0.23777018658851792) + | > loss_disc_real_3: 0.19986943900585175 (0.23570534709333812) + | > loss_disc_real_4: 0.23469646275043488 (0.23621153656585675) + | > loss_disc_real_5: 0.24389857053756714 (0.23793484669336132) + | > loss_0: 2.6963703632354736 (2.7637312692095986) + | > grad_norm_0: tensor(6.3772, device='cuda:0') (tensor(11.9780, device='cuda:0')) + | > loss_gen: 1.8738151788711548 (1.8654142688137825) + | > loss_kl: 2.4999470710754395 (2.5055264402443265) + | > loss_feat: 3.058316230773926 (2.94370456256777) + | > loss_mel: 22.63745880126953 (22.59304118716101) + | > loss_duration: 1.434131145477295 (1.3995443339079183) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.50366973876953 (31.307230806126842) + | > grad_norm_1: tensor(99.0661, device='cuda:0') (tensor(130.0163, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.9927 (0.883158954096512) + | > loader_time: 0.0187 (0.01337628465303233) + + + --> TIME: 2024-06-25 05:13:11 -- STEP: 451/591 -- GLOBAL_STEP: 8725 + | > loss_disc: 2.6936376094818115 (2.7633601567168995) + | > loss_disc_real_0: 0.2008863389492035 (0.21945126032551743) + | > loss_disc_real_1: 0.19378842413425446 (0.23840401395990152) + | > loss_disc_real_2: 0.20772086083889008 (0.237765220937602) + | > loss_disc_real_3: 0.21506130695343018 (0.23577992882406104) + | > loss_disc_real_4: 0.22127103805541992 (0.23609038695131332) + | > loss_disc_real_5: 0.262871116399765 (0.2378584938474875) + | > loss_0: 2.6936376094818115 (2.7633601567168995) + | > grad_norm_0: tensor(4.0605, device='cuda:0') (tensor(12.0574, device='cuda:0')) + | > loss_gen: 1.9502520561218262 (1.8646540200366677) + | > loss_kl: 2.4536519050598145 (2.502652230653954) + | > loss_feat: 3.0915815830230713 (2.9415687822185452) + | > loss_mel: 22.741310119628906 (22.575040322449677) + | > loss_duration: 1.414517879486084 (1.400447178838521) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.651315689086914 (31.284362564594414) + | > grad_norm_1: tensor(126.1307, device='cuda:0') (tensor(128.5066, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 0.9999 (0.8899686706568344) + | > loader_time: 0.0156 (0.013667028917705933) + + + --> TIME: 2024-06-25 05:13:37 -- STEP: 476/591 -- GLOBAL_STEP: 8750 + | > loss_disc: 2.777194023132324 (2.76319700128892) + | > loss_disc_real_0: 0.15094909071922302 (0.21957355513855686) + | > loss_disc_real_1: 0.19873523712158203 (0.23777059885133212) + | > loss_disc_real_2: 0.2471647411584854 (0.23780888074836812) + | > loss_disc_real_3: 0.24088680744171143 (0.23571750884797393) + | > loss_disc_real_4: 0.220284104347229 (0.23617189333719366) + | > loss_disc_real_5: 0.2044045329093933 (0.2378739333954178) + | > loss_0: 2.777194023132324 (2.76319700128892) + | > grad_norm_0: tensor(12.3252, device='cuda:0') (tensor(12.0960, device='cuda:0')) + | > loss_gen: 1.9214234352111816 (1.8640517943045671) + | > loss_kl: 2.387939453125 (2.5002289239098054) + | > loss_feat: 2.900641441345215 (2.9387141365964884) + | > loss_mel: 22.543716430664062 (22.559011054640056) + | > loss_duration: 1.4058661460876465 (1.4015524397377193) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.159587860107422 (31.263558395770417) + | > grad_norm_1: tensor(82.0396, device='cuda:0') (tensor(127.3672, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 1.07 (0.8970505119371814) + | > loader_time: 0.0195 (0.013990496887880214) + + + --> TIME: 2024-06-25 05:14:04 -- STEP: 501/591 -- GLOBAL_STEP: 8775 + | > loss_disc: 2.8017032146453857 (2.7628166651773367) + | > loss_disc_real_0: 0.269040048122406 (0.21978012635976726) + | > loss_disc_real_1: 0.2145889550447464 (0.23753227004985844) + | > loss_disc_real_2: 0.23441708087921143 (0.23778595812544376) + | > loss_disc_real_3: 0.28132978081703186 (0.23585179843707474) + | > loss_disc_real_4: 0.24014869332313538 (0.2361829521175392) + | > loss_disc_real_5: 0.2522006034851074 (0.23773637396132874) + | > loss_0: 2.8017032146453857 (2.7628166651773367) + | > grad_norm_0: tensor(10.2140, device='cuda:0') (tensor(11.8429, device='cuda:0')) + | > loss_gen: 1.7053065299987793 (1.8627819191671893) + | > loss_kl: 2.371878147125244 (2.4991707506769916) + | > loss_feat: 2.6723148822784424 (2.93502073611566) + | > loss_mel: 21.895233154296875 (22.536206738439624) + | > loss_duration: 1.4373538494110107 (1.4028611299758411) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.08208656311035 (31.23604131458762) + | > grad_norm_1: tensor(85.1036, device='cuda:0') (tensor(127.0496, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 1.0651 (0.9051480540734327) + | > loader_time: 0.0167 (0.014282293662339627) + + + --> TIME: 2024-06-25 05:14:33 -- STEP: 526/591 -- GLOBAL_STEP: 8800 + | > loss_disc: 2.833405017852783 (2.7628672780192858) + | > loss_disc_real_0: 0.17267677187919617 (0.21997606506422443) + | > loss_disc_real_1: 0.254716694355011 (0.23723648165568648) + | > loss_disc_real_2: 0.2529051601886749 (0.2377026948525425) + | > loss_disc_real_3: 0.24973994493484497 (0.23581277233452852) + | > loss_disc_real_4: 0.23762477934360504 (0.2361867647093965) + | > loss_disc_real_5: 0.25817954540252686 (0.23779376480062653) + | > loss_0: 2.833405017852783 (2.7628672780192858) + | > grad_norm_0: tensor(12.4034, device='cuda:0') (tensor(11.9700, device='cuda:0')) + | > loss_gen: 1.8052996397018433 (1.8621813122310567) + | > loss_kl: 2.473735809326172 (2.4959389601370234) + | > loss_feat: 2.6537485122680664 (2.9306063828812814) + | > loss_mel: 21.440000534057617 (22.516978789645) + | > loss_duration: 1.4297747611999512 (1.4044769728138407) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.802560806274414 (31.210182458275625) + | > grad_norm_1: tensor(74.1520, device='cuda:0') (tensor(126.1515, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 1.1058 (0.9147741205338743) + | > loader_time: 0.0217 (0.01458311670180056) + + + --> TIME: 2024-06-25 05:15:03 -- STEP: 551/591 -- GLOBAL_STEP: 8825 + | > loss_disc: 2.7176578044891357 (2.762177329747085) + | > loss_disc_real_0: 0.2751920819282532 (0.22001089370672586) + | > loss_disc_real_1: 0.23535557091236115 (0.23690676978517136) + | > loss_disc_real_2: 0.25210586190223694 (0.23775523443944657) + | > loss_disc_real_3: 0.224287211894989 (0.23588243770512826) + | > loss_disc_real_4: 0.24074192345142365 (0.23607800438486296) + | > loss_disc_real_5: 0.24827556312084198 (0.23791169030372114) + | > loss_0: 2.7176578044891357 (2.762177329747085) + | > grad_norm_0: tensor(13.4026, device='cuda:0') (tensor(11.8585, device='cuda:0')) + | > loss_gen: 1.6839271783828735 (1.8615790462753516) + | > loss_kl: 2.5113940238952637 (2.4942315070469023) + | > loss_feat: 2.823700428009033 (2.9272856470028423) + | > loss_mel: 22.36710548400879 (22.49765660239218) + | > loss_duration: 1.454160213470459 (1.4059453187966724) + | > amp_scaler: 1024.0 (531.5136116152448) + | > loss_1: 30.840288162231445 (31.186698165865863) + | > grad_norm_1: tensor(147.2589, device='cuda:0') (tensor(125.1234, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 1.1838 (0.9273225045247431) + | > loader_time: 0.0235 (0.014923040750022376) + + + --> TIME: 2024-06-25 05:15:35 -- STEP: 576/591 -- GLOBAL_STEP: 8850 + | > loss_disc: 2.7174031734466553 (2.7615344752040185) + | > loss_disc_real_0: 0.3042490482330322 (0.2200781336416387) + | > loss_disc_real_1: 0.2601819634437561 (0.2365149431603236) + | > loss_disc_real_2: 0.24951815605163574 (0.23768359070850742) + | > loss_disc_real_3: 0.25404101610183716 (0.23589706032847366) + | > loss_disc_real_4: 0.21814660727977753 (0.23594904388301075) + | > loss_disc_real_5: 0.2212146669626236 (0.23781519326277906) + | > loss_0: 2.7174031734466553 (2.7615344752040185) + | > grad_norm_0: tensor(17.3551, device='cuda:0') (tensor(11.9039, device='cuda:0')) + | > loss_gen: 1.6869630813598633 (1.8608411970651813) + | > loss_kl: 2.3477652072906494 (2.491038168470065) + | > loss_feat: 2.8531181812286377 (2.9222489289111566) + | > loss_mel: 21.48053550720215 (22.478674673371845) + | > loss_duration: 1.4542667865753174 (1.4076105964680496) + | > amp_scaler: 1024.0 (552.8888888888887) + | > loss_1: 29.822649002075195 (31.16041361292203) + | > grad_norm_1: tensor(50.2437, device='cuda:0') (tensor(124.5032, device='cuda:0')) + | > current_lr_0: 0.0001996502842328613 + | > current_lr_1: 0.0001996502842328613 + | > step_time: 1.3456 (0.9413057093818981) + | > loader_time: 0.0255 (0.015341944164699979) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036715852453353555 (-6.448968927910996e-05) + | > avg_loss_disc: 2.7755879341287817 (+0.02567292274312738) + | > avg_loss_disc_real_0: 0.21368608988345938 (+0.03535752378879711) + | > avg_loss_disc_real_1: 0.22980125850819527 (+0.09123021570291923) + | > avg_loss_disc_real_2: 0.20970371555774775 (+0.021532945493434402) + | > avg_loss_disc_real_3: 0.19642125545664035 (-0.051759670706505445) + | > avg_loss_disc_real_4: 0.20412524932242454 (-0.007267525221439114) + | > avg_loss_disc_real_5: 0.21072537531243993 (-0.029339020873637933) + | > avg_loss_0: 2.7755879341287817 (+0.02567292274312738) + | > avg_loss_gen: 1.6701052721510543 (+0.07394429216993625) + | > avg_loss_kl: 2.4759629533645957 (-0.08217146548818999) + | > avg_loss_feat: 2.980095036486362 (+0.04185583743643262) + | > avg_loss_mel: 22.42853286418509 (-0.1913028067730842) + | > avg_loss_duration: 1.3941016907387591 (-0.007759809494018777) + | > avg_loss_1: 30.94879751002535 (-0.16543420832207545) + + + > EPOCH: 16/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 05:16:02)  + + --> TIME: 2024-06-25 05:16:11 -- STEP: 10/591 -- GLOBAL_STEP: 8875 + | > loss_disc: 2.8031327724456787 (2.738472080230713) + | > loss_disc_real_0: 0.2577197551727295 (0.22548003643751144) + | > loss_disc_real_1: 0.2109391689300537 (0.21953098773956298) + | > loss_disc_real_2: 0.23432500660419464 (0.23301452100276948) + | > loss_disc_real_3: 0.25971153378486633 (0.25145419836044314) + | > loss_disc_real_4: 0.24277758598327637 (0.23407327383756638) + | > loss_disc_real_5: 0.26351800560951233 (0.2390982463955879) + | > loss_0: 2.8031327724456787 (2.738472080230713) + | > grad_norm_0: tensor(9.0101, device='cuda:0') (tensor(11.9358, device='cuda:0')) + | > loss_gen: 1.910386323928833 (1.8847792983055114) + | > loss_kl: 2.9232072830200195 (2.7044203758239744) + | > loss_feat: 2.900158166885376 (3.1684009552001955) + | > loss_mel: 22.695667266845703 (22.57989330291748) + | > loss_duration: 1.376235008239746 (1.37505761384964) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 31.805652618408203 (31.712550926208497) + | > grad_norm_1: tensor(85.9892, device='cuda:0') (tensor(90.1395, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8001 (0.7716078042984009) + | > loader_time: 0.0097 (0.008685684204101563) + + + --> TIME: 2024-06-25 05:16:32 -- STEP: 35/591 -- GLOBAL_STEP: 8900 + | > loss_disc: 2.7923340797424316 (2.7511161872318812) + | > loss_disc_real_0: 0.2152256816625595 (0.22207122423819134) + | > loss_disc_real_1: 0.2753174304962158 (0.23171023981911795) + | > loss_disc_real_2: 0.24924200773239136 (0.23325717832360948) + | > loss_disc_real_3: 0.27564555406570435 (0.24031832005296436) + | > loss_disc_real_4: 0.25007978081703186 (0.23848891130515507) + | > loss_disc_real_5: 0.2125312089920044 (0.23910968133381436) + | > loss_0: 2.7923340797424316 (2.7511161872318812) + | > grad_norm_0: tensor(4.7900, device='cuda:0') (tensor(11.9283, device='cuda:0')) + | > loss_gen: 1.7190425395965576 (1.873460360935756) + | > loss_kl: 2.5048885345458984 (2.6271488121577673) + | > loss_feat: 2.943901777267456 (3.1150116103036063) + | > loss_mel: 22.362293243408203 (22.49439522879464) + | > loss_duration: 1.349149227142334 (1.369805223601205) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 30.879276275634766 (31.47982128688267) + | > grad_norm_1: tensor(180.1703, device='cuda:0') (tensor(104.9975, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8031 (0.7838239874158587) + | > loader_time: 0.0095 (0.009220150538853236) + + + --> TIME: 2024-06-25 05:16:52 -- STEP: 60/591 -- GLOBAL_STEP: 8925 + | > loss_disc: 2.776641368865967 (2.74533097743988) + | > loss_disc_real_0: 0.20503634214401245 (0.22091600559651853) + | > loss_disc_real_1: 0.31303635239601135 (0.2308472141623497) + | > loss_disc_real_2: 0.22586354613304138 (0.23382265046238898) + | > loss_disc_real_3: 0.21866977214813232 (0.2379441075026989) + | > loss_disc_real_4: 0.22095276415348053 (0.23623792380094527) + | > loss_disc_real_5: 0.2280256301164627 (0.2384206602970759) + | > loss_0: 2.776641368865967 (2.74533097743988) + | > grad_norm_0: tensor(4.0313, device='cuda:0') (tensor(10.8943, device='cuda:0')) + | > loss_gen: 1.8527584075927734 (1.8730489571889242) + | > loss_kl: 2.5279881954193115 (2.592907253901164) + | > loss_feat: 3.0569355487823486 (3.097784423828125) + | > loss_mel: 21.881248474121094 (22.4695319811503) + | > loss_duration: 1.4373342990875244 (1.3806446592013042) + | > amp_scaler: 512.0 (981.3333333333334) + | > loss_1: 30.756263732910156 (31.41391741434733) + | > grad_norm_1: tensor(93.1538, device='cuda:0') (tensor(104.9502, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8094 (0.7928868730862936) + | > loader_time: 0.01 (0.009447399775187175) + + + --> TIME: 2024-06-25 05:17:13 -- STEP: 85/591 -- GLOBAL_STEP: 8950 + | > loss_disc: 2.7027132511138916 (2.743007769304163) + | > loss_disc_real_0: 0.20073698461055756 (0.21902214385130825) + | > loss_disc_real_1: 0.2659079134464264 (0.23047251192962423) + | > loss_disc_real_2: 0.21572159230709076 (0.23404844508451575) + | > loss_disc_real_3: 0.19597001373767853 (0.23785496389164645) + | > loss_disc_real_4: 0.21432840824127197 (0.23615250973140492) + | > loss_disc_real_5: 0.19263198971748352 (0.23795870279564577) + | > loss_0: 2.7027132511138916 (2.743007769304163) + | > grad_norm_0: tensor(4.8853, device='cuda:0') (tensor(10.6706, device='cuda:0')) + | > loss_gen: 1.8555328845977783 (1.8699050244163065) + | > loss_kl: 2.4560365676879883 (2.56645235454335) + | > loss_feat: 3.0218982696533203 (3.085402645784266) + | > loss_mel: 22.59830093383789 (22.45189653284409) + | > loss_duration: 1.44199800491333 (1.3883810716516833) + | > amp_scaler: 512.0 (843.2941176470588) + | > loss_1: 31.373767852783203 (31.362037658691406) + | > grad_norm_1: tensor(197.5820, device='cuda:0') (tensor(114.2320, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8322 (0.8044269561767579) + | > loader_time: 0.0113 (0.009799704832189221) + + + --> TIME: 2024-06-25 05:17:35 -- STEP: 110/591 -- GLOBAL_STEP: 8975 + | > loss_disc: 2.7492926120758057 (2.74433173266324) + | > loss_disc_real_0: 0.3184289336204529 (0.21949886571277272) + | > loss_disc_real_1: 0.24665643274784088 (0.22961441468108784) + | > loss_disc_real_2: 0.22667080163955688 (0.23406391468915072) + | > loss_disc_real_3: 0.2326132357120514 (0.23754829466342925) + | > loss_disc_real_4: 0.28707030415534973 (0.23668868162415244) + | > loss_disc_real_5: 0.2423401027917862 (0.23852667253125798) + | > loss_0: 2.7492926120758057 (2.74433173266324) + | > grad_norm_0: tensor(17.9047, device='cuda:0') (tensor(11.1133, device='cuda:0')) + | > loss_gen: 1.8237398862838745 (1.8711156205697492) + | > loss_kl: 2.553574323654175 (2.5497424429113225) + | > loss_feat: 3.0380702018737793 (3.0824770407243207) + | > loss_mel: 22.211326599121094 (22.44994231137362) + | > loss_duration: 1.386649489402771 (1.3931100541895085) + | > amp_scaler: 512.0 (768.0) + | > loss_1: 31.01336097717285 (31.34638751636852) + | > grad_norm_1: tensor(108.4914, device='cuda:0') (tensor(118.4971, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8275 (0.8115548155524513) + | > loader_time: 0.0105 (0.010085105895996097) + + + --> TIME: 2024-06-25 05:17:56 -- STEP: 135/591 -- GLOBAL_STEP: 9000 + | > loss_disc: 2.752190589904785 (2.742686503021805) + | > loss_disc_real_0: 0.2635458707809448 (0.2192999028497272) + | > loss_disc_real_1: 0.21004845201969147 (0.22947811627829517) + | > loss_disc_real_2: 0.25187087059020996 (0.23468823421884466) + | > loss_disc_real_3: 0.2544119656085968 (0.23687080476019118) + | > loss_disc_real_4: 0.21235711872577667 (0.23594629367192585) + | > loss_disc_real_5: 0.19972634315490723 (0.2380351119571262) + | > loss_0: 2.752190589904785 (2.742686503021805) + | > grad_norm_0: tensor(14.2227, device='cuda:0') (tensor(11.2101, device='cuda:0')) + | > loss_gen: 1.724367618560791 (1.8714548614290025) + | > loss_kl: 2.41994309425354 (2.536334933174982) + | > loss_feat: 3.0099446773529053 (3.08567773854291) + | > loss_mel: 22.470993041992188 (22.45922804938422) + | > loss_duration: 1.3657288551330566 (1.3937321247877899) + | > amp_scaler: 512.0 (720.5925925925926) + | > loss_1: 30.990976333618164 (31.346427677295825) + | > grad_norm_1: tensor(44.1319, device='cuda:0') (tensor(118.7485, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8454 (0.817609656298602) + | > loader_time: 0.0117 (0.010347599453396271) + + + --> TIME: 2024-06-25 05:18:18 -- STEP: 160/591 -- GLOBAL_STEP: 9025 + | > loss_disc: 2.7368016242980957 (2.743937255442142) + | > loss_disc_real_0: 0.21377190947532654 (0.21981426049023867) + | > loss_disc_real_1: 0.19753453135490417 (0.229330278839916) + | > loss_disc_real_2: 0.2493649125099182 (0.2347083931788802) + | > loss_disc_real_3: 0.23714296519756317 (0.2368084393441677) + | > loss_disc_real_4: 0.23669913411140442 (0.23606975497677923) + | > loss_disc_real_5: 0.25048959255218506 (0.23821814358234406) + | > loss_0: 2.7368016242980957 (2.743937255442142) + | > grad_norm_0: tensor(4.2708, device='cuda:0') (tensor(11.7991, device='cuda:0')) + | > loss_gen: 1.8781954050064087 (1.8714436650276185) + | > loss_kl: 2.403644561767578 (2.531652002036572) + | > loss_feat: 2.984243392944336 (3.080475494265557) + | > loss_mel: 22.69976806640625 (22.420343279838562) + | > loss_duration: 1.3700497150421143 (1.3918730176985263) + | > amp_scaler: 512.0 (688.0) + | > loss_1: 31.335901260375977 (31.29578741788864) + | > grad_norm_1: tensor(47.6785, device='cuda:0') (tensor(114.5200, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8521 (0.8236705139279366) + | > loader_time: 0.0116 (0.010617402195930485) + + + --> TIME: 2024-06-25 05:18:40 -- STEP: 185/591 -- GLOBAL_STEP: 9050 + | > loss_disc: 2.7363996505737305 (2.743189546224232) + | > loss_disc_real_0: 0.24750766158103943 (0.22037112451082952) + | > loss_disc_real_1: 0.23279176652431488 (0.2289398389088141) + | > loss_disc_real_2: 0.24435964226722717 (0.23454873682679356) + | > loss_disc_real_3: 0.24371838569641113 (0.23649801410533286) + | > loss_disc_real_4: 0.24839279055595398 (0.23589187731613984) + | > loss_disc_real_5: 0.2536548972129822 (0.23815207513603004) + | > loss_0: 2.7363996505737305 (2.743189546224232) + | > grad_norm_0: tensor(4.8254, device='cuda:0') (tensor(12.1375, device='cuda:0')) + | > loss_gen: 2.006967306137085 (1.873466964025755) + | > loss_kl: 2.6695330142974854 (2.5283222108273895) + | > loss_feat: 2.9351918697357178 (3.0838502651936297) + | > loss_mel: 22.05563735961914 (22.413155375300217) + | > loss_duration: 1.402681589126587 (1.3904057373871668) + | > amp_scaler: 512.0 (664.2162162162161) + | > loss_1: 31.070011138916016 (31.289200550801045) + | > grad_norm_1: tensor(224.4179, device='cuda:0') (tensor(113.8546, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8598 (0.8294777960390658) + | > loader_time: 0.012 (0.010833112613574888) + + + --> TIME: 2024-06-25 05:19:02 -- STEP: 210/591 -- GLOBAL_STEP: 9075 + | > loss_disc: 2.6505887508392334 (2.7422310522624413) + | > loss_disc_real_0: 0.13764017820358276 (0.21988315046543166) + | > loss_disc_real_1: 0.18866920471191406 (0.2286714772383372) + | > loss_disc_real_2: 0.2164759784936905 (0.23468660293590454) + | > loss_disc_real_3: 0.257500559091568 (0.2367158742887633) + | > loss_disc_real_4: 0.19763602316379547 (0.2356903877286684) + | > loss_disc_real_5: 0.20952260494232178 (0.23826519428264528) + | > loss_0: 2.6505887508392334 (2.7422310522624413) + | > grad_norm_0: tensor(16.9301, device='cuda:0') (tensor(11.7483, device='cuda:0')) + | > loss_gen: 1.9557058811187744 (1.8743786948067802) + | > loss_kl: 2.50030517578125 (2.5226118382953464) + | > loss_feat: 3.052990198135376 (3.082643149012611) + | > loss_mel: 22.76498794555664 (22.421062269664937) + | > loss_duration: 1.3975412845611572 (1.3895813096137268) + | > amp_scaler: 512.0 (646.0952380952378) + | > loss_1: 31.67152976989746 (31.290277254013787) + | > grad_norm_1: tensor(194.1245, device='cuda:0') (tensor(120.8857, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8769 (0.8345331021717617) + | > loader_time: 0.0146 (0.01109923067547027) + + + --> TIME: 2024-06-25 05:19:25 -- STEP: 235/591 -- GLOBAL_STEP: 9100 + | > loss_disc: 2.777801990509033 (2.7411425844151913) + | > loss_disc_real_0: 0.25440114736557007 (0.21950602667762878) + | > loss_disc_real_1: 0.19850876927375793 (0.228599473263355) + | > loss_disc_real_2: 0.23315194249153137 (0.2349215898742067) + | > loss_disc_real_3: 0.19428789615631104 (0.23628751198027997) + | > loss_disc_real_4: 0.23874200880527496 (0.23571160721017959) + | > loss_disc_real_5: 0.2289257049560547 (0.23805144093138106) + | > loss_0: 2.777801990509033 (2.7411425844151913) + | > grad_norm_0: tensor(8.5241, device='cuda:0') (tensor(11.5570, device='cuda:0')) + | > loss_gen: 1.8962671756744385 (1.873735170668744) + | > loss_kl: 2.5153000354766846 (2.5194609956538425) + | > loss_feat: 3.0866899490356445 (3.079839375678529) + | > loss_mel: 22.42500877380371 (22.414163345986218) + | > loss_duration: 1.360077977180481 (1.389137017473261) + | > amp_scaler: 512.0 (631.8297872340423) + | > loss_1: 31.283344268798828 (31.276335931331552) + | > grad_norm_1: tensor(93.5318, device='cuda:0') (tensor(118.9613, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8807 (0.8390599808794387) + | > loader_time: 0.0158 (0.01143249653755351) + + + --> TIME: 2024-06-25 05:19:48 -- STEP: 260/591 -- GLOBAL_STEP: 9125 + | > loss_disc: 2.810210943222046 (2.741403225752023) + | > loss_disc_real_0: 0.2162654995918274 (0.21960365500014561) + | > loss_disc_real_1: 0.26034072041511536 (0.2284757701250223) + | > loss_disc_real_2: 0.21305397152900696 (0.23510227627479113) + | > loss_disc_real_3: 0.24065552651882172 (0.2364540672646119) + | > loss_disc_real_4: 0.22308003902435303 (0.2356581694804705) + | > loss_disc_real_5: 0.2515665888786316 (0.23810626767002618) + | > loss_0: 2.810210943222046 (2.741403225752023) + | > grad_norm_0: tensor(5.2120, device='cuda:0') (tensor(11.6598, device='cuda:0')) + | > loss_gen: 1.8127362728118896 (1.874232984506167) + | > loss_kl: 2.5588815212249756 (2.5183581398083614) + | > loss_feat: 2.9888155460357666 (3.077694695729476) + | > loss_mel: 22.44426727294922 (22.423714380997872) + | > loss_duration: 1.3810380697250366 (1.3887705000547257) + | > amp_scaler: 512.0 (620.307692307692) + | > loss_1: 31.185739517211914 (31.282770721728983) + | > grad_norm_1: tensor(109.2201, device='cuda:0') (tensor(119.4099, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.8913 (0.8446549919935373) + | > loader_time: 0.0143 (0.011706564059624312) + + + --> TIME: 2024-06-25 05:20:10 -- STEP: 285/591 -- GLOBAL_STEP: 9150 + | > loss_disc: 2.8028900623321533 (2.7416580016152894) + | > loss_disc_real_0: 0.317828893661499 (0.21989013735662427) + | > loss_disc_real_1: 0.31781986355781555 (0.22874194143111246) + | > loss_disc_real_2: 0.2650424838066101 (0.23515014564781858) + | > loss_disc_real_3: 0.21079491078853607 (0.23635163260133643) + | > loss_disc_real_4: 0.25471407175064087 (0.23552614659593815) + | > loss_disc_real_5: 0.25151294469833374 (0.23782450755437215) + | > loss_0: 2.8028900623321533 (2.7416580016152894) + | > grad_norm_0: tensor(12.6231, device='cuda:0') (tensor(11.6848, device='cuda:0')) + | > loss_gen: 1.8228493928909302 (1.8739475961317096) + | > loss_kl: 2.544668436050415 (2.51426851540281) + | > loss_feat: 3.0101590156555176 (3.0719205864688806) + | > loss_mel: 22.187284469604492 (22.41386222839355) + | > loss_duration: 1.3849115371704102 (1.389337139380605) + | > amp_scaler: 512.0 (610.8070175438595) + | > loss_1: 30.949871063232422 (31.263336074561405) + | > grad_norm_1: tensor(73.3513, device='cuda:0') (tensor(119.2743, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.9096 (0.8494393992842293) + | > loader_time: 0.0138 (0.01197537957576284) + + + --> TIME: 2024-06-25 05:20:34 -- STEP: 310/591 -- GLOBAL_STEP: 9175 + | > loss_disc: 2.8354294300079346 (2.7421007879318724) + | > loss_disc_real_0: 0.21252137422561646 (0.21974430021739774) + | > loss_disc_real_1: 0.22376678884029388 (0.2287787425902582) + | > loss_disc_real_2: 0.22825823724269867 (0.23510093953340283) + | > loss_disc_real_3: 0.1874094158411026 (0.23616901775521618) + | > loss_disc_real_4: 0.23419153690338135 (0.23545563759342317) + | > loss_disc_real_5: 0.22158432006835938 (0.23771792602154515) + | > loss_0: 2.8354294300079346 (2.7421007879318724) + | > grad_norm_0: tensor(4.8592, device='cuda:0') (tensor(11.6044, device='cuda:0')) + | > loss_gen: 2.0531115531921387 (1.8739805609949174) + | > loss_kl: 2.478604316711426 (2.5134837588956276) + | > loss_feat: 3.227853536605835 (3.06755425853114) + | > loss_mel: 22.163501739501953 (22.419053729887935) + | > loss_duration: 1.395734429359436 (1.3898102887215145) + | > amp_scaler: 512.0 (602.8387096774192) + | > loss_1: 31.31880760192871 (31.2638826001075) + | > grad_norm_1: tensor(39.4873, device='cuda:0') (tensor(123.1104, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.9166 (0.8551078611804597) + | > loader_time: 0.0165 (0.012209549257832195) + + + --> TIME: 2024-06-25 05:20:58 -- STEP: 335/591 -- GLOBAL_STEP: 9200 + | > loss_disc: 2.7502212524414062 (2.7419979358786963) + | > loss_disc_real_0: 0.2609942555427551 (0.2199964171914912) + | > loss_disc_real_1: 0.23130053281784058 (0.22871865407744452) + | > loss_disc_real_2: 0.29050371050834656 (0.23516548160296769) + | > loss_disc_real_3: 0.23872387409210205 (0.23644442923033415) + | > loss_disc_real_4: 0.21281753480434418 (0.23525461025202452) + | > loss_disc_real_5: 0.24125784635543823 (0.23771158932749906) + | > loss_0: 2.7502212524414062 (2.7419979358786963) + | > grad_norm_0: tensor(9.1124, device='cuda:0') (tensor(11.7564, device='cuda:0')) + | > loss_gen: 1.8133395910263062 (1.8732782951041833) + | > loss_kl: 2.479776620864868 (2.508562226793659) + | > loss_feat: 2.9811127185821533 (3.0628006714493496) + | > loss_mel: 22.78068733215332 (22.402937060683517) + | > loss_duration: 1.4123973846435547 (1.3904537108407082) + | > amp_scaler: 512.0 (596.0597014925373) + | > loss_1: 31.467313766479492 (31.238031945299745) + | > grad_norm_1: tensor(52.0495, device='cuda:0') (tensor(122.0362, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.9485 (0.8606316516648481) + | > loader_time: 0.0202 (0.012544257605253765) + + + --> TIME: 2024-06-25 05:21:23 -- STEP: 360/591 -- GLOBAL_STEP: 9225 + | > loss_disc: 2.7299466133117676 (2.741675009992387) + | > loss_disc_real_0: 0.23926818370819092 (0.21956969387829303) + | > loss_disc_real_1: 0.1890193521976471 (0.2287116045753161) + | > loss_disc_real_2: 0.21213990449905396 (0.23536364278859562) + | > loss_disc_real_3: 0.18820376694202423 (0.23642136189672683) + | > loss_disc_real_4: 0.216033473610878 (0.23531822036537858) + | > loss_disc_real_5: 0.24519053101539612 (0.23767857191463312) + | > loss_0: 2.7299466133117676 (2.741675009992387) + | > grad_norm_0: tensor(7.9299, device='cuda:0') (tensor(11.6092, device='cuda:0')) + | > loss_gen: 1.9025615453720093 (1.8731866829925112) + | > loss_kl: 2.4950075149536133 (2.5045283006297216) + | > loss_feat: 3.1608219146728516 (3.0591361443201714) + | > loss_mel: 22.250967025756836 (22.383688863118497) + | > loss_duration: 1.3947117328643799 (1.3918415947092897) + | > amp_scaler: 512.0 (590.222222222222) + | > loss_1: 31.204069137573242 (31.212381580140857) + | > grad_norm_1: tensor(280.3253, device='cuda:0') (tensor(124.9757, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.9579 (0.8693226973215742) + | > loader_time: 0.0145 (0.013383973307079744) + + + --> TIME: 2024-06-25 05:21:48 -- STEP: 385/591 -- GLOBAL_STEP: 9250 + | > loss_disc: 3.0720620155334473 (2.7478590203570077) + | > loss_disc_real_0: 0.33080464601516724 (0.2199631192660951) + | > loss_disc_real_1: 0.4472728669643402 (0.2311737865403101) + | > loss_disc_real_2: 0.3428659439086914 (0.23603781326250597) + | > loss_disc_real_3: 0.2360731065273285 (0.23667766246702765) + | > loss_disc_real_4: 0.2731271982192993 (0.2355875420880008) + | > loss_disc_real_5: 0.21770736575126648 (0.23775298792820473) + | > loss_0: 3.0720620155334473 (2.7478590203570077) + | > grad_norm_0: tensor(19.8165, device='cuda:0') (tensor(11.8631, device='cuda:0')) + | > loss_gen: 1.8183990716934204 (1.8762042218988593) + | > loss_kl: 2.5015885829925537 (2.5034218565210105) + | > loss_feat: 2.701683521270752 (3.0532267700542115) + | > loss_mel: 23.04921531677246 (22.39019384012595) + | > loss_duration: 1.4106463193893433 (1.3933371621292905) + | > amp_scaler: 512.0 (585.142857142857) + | > loss_1: 31.48153305053711 (31.216383857231637) + | > grad_norm_1: tensor(108.6648, device='cuda:0') (tensor(129.5947, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.9678 (0.8751143963305984) + | > loader_time: 0.0167 (0.013671237152892278) + + + --> TIME: 2024-06-25 05:22:12 -- STEP: 410/591 -- GLOBAL_STEP: 9275 + | > loss_disc: 2.7527811527252197 (2.7534369073263023) + | > loss_disc_real_0: 0.3386087417602539 (0.22011271535259921) + | > loss_disc_real_1: 0.2523726522922516 (0.23308759398940132) + | > loss_disc_real_2: 0.2212248146533966 (0.237026423942752) + | > loss_disc_real_3: 0.23785854876041412 (0.2365928679704666) + | > loss_disc_real_4: 0.22881488502025604 (0.2358745738137059) + | > loss_disc_real_5: 0.23007763922214508 (0.23770657450687593) + | > loss_0: 2.7527811527252197 (2.7534369073263023) + | > grad_norm_0: tensor(27.8431, device='cuda:0') (tensor(11.8231, device='cuda:0')) + | > loss_gen: 1.711463451385498 (1.8725082321864803) + | > loss_kl: 2.362338066101074 (2.5002750012932733) + | > loss_feat: 2.750185012817383 (3.032395924009929) + | > loss_mel: 22.18954849243164 (22.383766751173074) + | > loss_duration: 1.42913818359375 (1.3939472681138572) + | > amp_scaler: 512.0 (580.6829268292682) + | > loss_1: 30.442672729492188 (31.18289317154303) + | > grad_norm_1: tensor(68.3344, device='cuda:0') (tensor(129.1105, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 0.9685 (0.8807282913021928) + | > loader_time: 0.019 (0.013913279626427631) + + + --> TIME: 2024-06-25 05:22:38 -- STEP: 435/591 -- GLOBAL_STEP: 9300 + | > loss_disc: 2.7500712871551514 (2.7528236553586756) + | > loss_disc_real_0: 0.1668984591960907 (0.21974290131494917) + | > loss_disc_real_1: 0.2063487023115158 (0.23286515955609838) + | > loss_disc_real_2: 0.22022415697574615 (0.23694296215457478) + | > loss_disc_real_3: 0.2345086634159088 (0.23654802934876804) + | > loss_disc_real_4: 0.24411305785179138 (0.2359072483476551) + | > loss_disc_real_5: 0.24002857506275177 (0.2378416697184245) + | > loss_0: 2.7500712871551514 (2.7528236553586756) + | > grad_norm_0: tensor(16.2515, device='cuda:0') (tensor(11.7205, device='cuda:0')) + | > loss_gen: 1.844823956489563 (1.870593166077274) + | > loss_kl: 2.4301862716674805 (2.49691954261955) + | > loss_feat: 2.9746310710906982 (3.0243456089633645) + | > loss_mel: 21.639007568359375 (22.36627290221467) + | > loss_duration: 1.412729024887085 (1.394751562743351) + | > amp_scaler: 512.0 (576.735632183908) + | > loss_1: 30.30137825012207 (31.152882753569504) + | > grad_norm_1: tensor(171.2160, device='cuda:0') (tensor(127.5428, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 1.0068 (0.8869936537468573) + | > loader_time: 0.021 (0.014149520040928636) + + + --> TIME: 2024-06-25 05:23:04 -- STEP: 460/591 -- GLOBAL_STEP: 9325 + | > loss_disc: 2.740056276321411 (2.7526227681533144) + | > loss_disc_real_0: 0.1629638671875 (0.21967529222045257) + | > loss_disc_real_1: 0.22985924780368805 (0.23257182860827963) + | > loss_disc_real_2: 0.23540867865085602 (0.23687891014244244) + | > loss_disc_real_3: 0.2594442069530487 (0.23662960720450982) + | > loss_disc_real_4: 0.20797716081142426 (0.23584365938668667) + | > loss_disc_real_5: 0.19854360818862915 (0.2378798641588377) + | > loss_0: 2.740056276321411 (2.7526227681533144) + | > grad_norm_0: tensor(18.4042, device='cuda:0') (tensor(11.6877, device='cuda:0')) + | > loss_gen: 1.8870753049850464 (1.8697365156982255) + | > loss_kl: 2.419664144515991 (2.495350087725596) + | > loss_feat: 2.9501760005950928 (3.019027154341989) + | > loss_mel: 22.641382217407227 (22.352594321707027) + | > loss_duration: 1.426347255706787 (1.3956985973793525) + | > amp_scaler: 512.0 (573.2173913043476) + | > loss_1: 31.32464599609375 (31.13240666596786) + | > grad_norm_1: tensor(116.3675, device='cuda:0') (tensor(126.6626, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 1.0318 (0.8942403414975044) + | > loader_time: 0.0194 (0.014408063888549809) + + + --> TIME: 2024-06-25 05:23:30 -- STEP: 485/591 -- GLOBAL_STEP: 9350 + | > loss_disc: 2.6798291206359863 (2.7517526567596744) + | > loss_disc_real_0: 0.11257782578468323 (0.21957669616052786) + | > loss_disc_real_1: 0.18349184095859528 (0.23227465369344988) + | > loss_disc_real_2: 0.2515699863433838 (0.23684884272899825) + | > loss_disc_real_3: 0.20813199877738953 (0.2364124718400621) + | > loss_disc_real_4: 0.1980365514755249 (0.2357594465779275) + | > loss_disc_real_5: 0.19984015822410583 (0.23784009101464576) + | > loss_0: 2.6798291206359863 (2.7517526567596744) + | > grad_norm_0: tensor(28.8654, device='cuda:0') (tensor(11.6138, device='cuda:0')) + | > loss_gen: 2.0495452880859375 (1.8698468306629927) + | > loss_kl: 2.5102920532226562 (2.4943100796532365) + | > loss_feat: 3.134037971496582 (3.015719351817653) + | > loss_mel: 22.022201538085938 (22.342324931351186) + | > loss_duration: 1.4286067485809326 (1.3971621331480357) + | > amp_scaler: 512.0 (570.061855670103) + | > loss_1: 31.144681930541992 (31.119363312868728) + | > grad_norm_1: tensor(62.3777, device='cuda:0') (tensor(125.4873, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 1.0216 (0.9018231067460839) + | > loader_time: 0.0201 (0.01466786836840443) + + + --> TIME: 2024-06-25 05:23:58 -- STEP: 510/591 -- GLOBAL_STEP: 9375 + | > loss_disc: 2.8097403049468994 (2.752324486713783) + | > loss_disc_real_0: 0.14565378427505493 (0.21986713784874654) + | > loss_disc_real_1: 0.24384956061840057 (0.23215540882711316) + | > loss_disc_real_2: 0.2788604497909546 (0.23702891267397824) + | > loss_disc_real_3: 0.2995263338088989 (0.23649902711896337) + | > loss_disc_real_4: 0.24247361719608307 (0.2357067894117505) + | > loss_disc_real_5: 0.23200951516628265 (0.2379112412239991) + | > loss_0: 2.8097403049468994 (2.752324486713783) + | > grad_norm_0: tensor(18.2179, device='cuda:0') (tensor(11.6631, device='cuda:0')) + | > loss_gen: 1.7717007398605347 (1.869281558896981) + | > loss_kl: 2.4859867095947266 (2.4927654037288574) + | > loss_feat: 2.716118097305298 (3.010307579414518) + | > loss_mel: 22.2589054107666 (22.324857640733914) + | > loss_duration: 1.4107418060302734 (1.398568868403341) + | > amp_scaler: 512.0 (567.2156862745097) + | > loss_1: 30.643451690673828 (31.0957810458015) + | > grad_norm_1: tensor(74.4028, device='cuda:0') (tensor(124.6588, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 1.0609 (0.9108872334162397) + | > loader_time: 0.0186 (0.014954210730159987) + + + --> TIME: 2024-06-25 05:24:27 -- STEP: 535/591 -- GLOBAL_STEP: 9400 + | > loss_disc: 2.7142133712768555 (2.751890198092594) + | > loss_disc_real_0: 0.23956631124019623 (0.219786619847623) + | > loss_disc_real_1: 0.1990673691034317 (0.2320091118740144) + | > loss_disc_real_2: 0.26393353939056396 (0.23704605949259247) + | > loss_disc_real_3: 0.19868573546409607 (0.2364753824806659) + | > loss_disc_real_4: 0.22917716205120087 (0.23568104597452644) + | > loss_disc_real_5: 0.24347913265228271 (0.23785747999899856) + | > loss_0: 2.7142133712768555 (2.751890198092594) + | > grad_norm_0: tensor(10.4120, device='cuda:0') (tensor(11.5793, device='cuda:0')) + | > loss_gen: 1.7437046766281128 (1.868404806885764) + | > loss_kl: 2.365767240524292 (2.4897577611085393) + | > loss_feat: 2.774832248687744 (3.00471446759233) + | > loss_mel: 21.491195678710938 (22.308312268123455) + | > loss_duration: 1.396108865737915 (1.4000256923871617) + | > amp_scaler: 512.0 (564.6355140186915) + | > loss_1: 29.7716064453125 (31.071214996765708) + | > grad_norm_1: tensor(58.0322, device='cuda:0') (tensor(125.0995, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 1.1658 (0.9215346416580344) + | > loader_time: 0.0231 (0.015257893321670107) + + + --> TIME: 2024-06-25 05:24:59 -- STEP: 560/591 -- GLOBAL_STEP: 9425 + | > loss_disc: 2.7718567848205566 (2.7523026006562383) + | > loss_disc_real_0: 0.2304564118385315 (0.22000755806719616) + | > loss_disc_real_1: 0.24023598432540894 (0.23193923834977406) + | > loss_disc_real_2: 0.26877152919769287 (0.23705631255039147) + | > loss_disc_real_3: 0.2545689642429352 (0.23639732655137777) + | > loss_disc_real_4: 0.24664971232414246 (0.2356258396591459) + | > loss_disc_real_5: 0.2577449083328247 (0.2377336549971785) + | > loss_0: 2.7718567848205566 (2.7523026006562383) + | > grad_norm_0: tensor(5.3927, device='cuda:0') (tensor(11.5912, device='cuda:0')) + | > loss_gen: 1.9102187156677246 (1.8676156474011285) + | > loss_kl: 2.5303492546081543 (2.488186564615793) + | > loss_feat: 2.8169350624084473 (2.9981279509408156) + | > loss_mel: 21.89556312561035 (22.28882475580488) + | > loss_duration: 1.4321088790893555 (1.4017437617693624) + | > amp_scaler: 512.0 (562.2857142857142) + | > loss_1: 30.585174560546875 (31.044498682022095) + | > grad_norm_1: tensor(163.0567, device='cuda:0') (tensor(124.8381, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 1.2962 (0.9347946460757937) + | > loader_time: 0.0257 (0.015614081280572076) + + + --> TIME: 2024-06-25 05:25:32 -- STEP: 585/591 -- GLOBAL_STEP: 9450 + | > loss_disc: 2.8396902084350586 (2.752274024180878) + | > loss_disc_real_0: 0.27768683433532715 (0.22004594958872878) + | > loss_disc_real_1: 0.2784487009048462 (0.23190220959930338) + | > loss_disc_real_2: 0.2074481099843979 (0.23691052180579586) + | > loss_disc_real_3: 0.20510542392730713 (0.2363463900792293) + | > loss_disc_real_4: 0.24162693321704865 (0.23562217803082913) + | > loss_disc_real_5: 0.24768805503845215 (0.23775462531126462) + | > loss_0: 2.8396902084350586 (2.752274024180878) + | > grad_norm_0: tensor(14.8039, device='cuda:0') (tensor(11.6015, device='cuda:0')) + | > loss_gen: 1.7367545366287231 (1.8674048407464965) + | > loss_kl: 2.435230016708374 (2.485229489742181) + | > loss_feat: 2.561753273010254 (2.993690622362319) + | > loss_mel: 21.10590934753418 (22.268283762483527) + | > loss_duration: 1.4490716457366943 (1.403359066930591) + | > amp_scaler: 512.0 (560.1367521367521) + | > loss_1: 29.288721084594727 (31.017967784914195) + | > grad_norm_1: tensor(70.0059, device='cuda:0') (tensor(124.4327, device='cuda:0')) + | > current_lr_0: 0.00019962532794733217 + | > current_lr_1: 0.00019962532794733217 + | > step_time: 1.3264 (0.9503906376341469) + | > loader_time: 0.0225 (0.016014893442137634) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003613791567213992 (-5.779367812136374e-05) + | > avg_loss_disc: 2.777798825121941 (+0.00221089099315952) + | > avg_loss_disc_real_0: 0.18849826619980184 (-0.025187823683657545) + | > avg_loss_disc_real_1: 0.23915406427484878 (+0.00935280576665351) + | > avg_loss_disc_real_2: 0.29533957420511453 (+0.08563585864736678) + | > avg_loss_disc_real_3: 0.23065809112914065 (+0.0342368356725003) + | > avg_loss_disc_real_4: 0.25976180332772275 (+0.05563655400529821) + | > avg_loss_disc_real_5: 0.2715436924011149 (+0.060818317088674995) + | > avg_loss_0: 2.777798825121941 (+0.00221089099315952) + | > avg_loss_gen: 1.8753650188446045 (+0.20525974669355018) + | > avg_loss_kl: 2.4528704501212912 (-0.023092503243304474) + | > avg_loss_feat: 2.867064902122984 (-0.11303013436337794) + | > avg_loss_mel: 22.122046937333778 (-0.3064859268513125) + | > avg_loss_duration: 1.3978507442677275 (+0.0037490535289683713) + | > avg_loss_1: 30.715197948699302 (-0.23359956132604864) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_9456.pth + + > EPOCH: 17/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 05:25:52)  + + --> TIME: 2024-06-25 05:26:09 -- STEP: 19/591 -- GLOBAL_STEP: 9475 + | > loss_disc: 2.703944206237793 (2.7355629268445467) + | > loss_disc_real_0: 0.26278433203697205 (0.22098508751706072) + | > loss_disc_real_1: 0.21614836156368256 (0.23081154180200478) + | > loss_disc_real_2: 0.2037172019481659 (0.2331271265682421) + | > loss_disc_real_3: 0.2684284448623657 (0.23488612865146838) + | > loss_disc_real_4: 0.2113126516342163 (0.236659395851587) + | > loss_disc_real_5: 0.21963384747505188 (0.2382679338517942) + | > loss_0: 2.703944206237793 (2.7355629268445467) + | > grad_norm_0: tensor(10.4488, device='cuda:0') (tensor(13.9228, device='cuda:0')) + | > loss_gen: 1.875734567642212 (1.8881759455329494) + | > loss_kl: 2.627577543258667 (2.6848810597469934) + | > loss_feat: 3.216221570968628 (3.2332959677043713) + | > loss_mel: 22.160240173339844 (22.410654268766706) + | > loss_duration: 1.3623504638671875 (1.359272040818867) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.242124557495117 (31.57627918845729) + | > grad_norm_1: tensor(127.8009, device='cuda:0') (tensor(112.7027, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.7788 (0.7752342851538407) + | > loader_time: 0.0095 (0.008682188234831157) + + + --> TIME: 2024-06-25 05:26:29 -- STEP: 44/591 -- GLOBAL_STEP: 9500 + | > loss_disc: 2.699997663497925 (2.728572233156725) + | > loss_disc_real_0: 0.3078727722167969 (0.21867630038071761) + | > loss_disc_real_1: 0.17859740555286407 (0.228522544557398) + | > loss_disc_real_2: 0.20904609560966492 (0.23187885670499367) + | > loss_disc_real_3: 0.23556512594223022 (0.23411689258434557) + | > loss_disc_real_4: 0.20869095623493195 (0.23576613854278217) + | > loss_disc_real_5: 0.22131797671318054 (0.23767355694012207) + | > loss_0: 2.699997663497925 (2.728572233156725) + | > grad_norm_0: tensor(16.8678, device='cuda:0') (tensor(12.5632, device='cuda:0')) + | > loss_gen: 1.8658571243286133 (1.882931717417457) + | > loss_kl: 2.6382811069488525 (2.620476923205636) + | > loss_feat: 3.3092644214630127 (3.2001094709743154) + | > loss_mel: 22.29627227783203 (22.332755955782805) + | > loss_duration: 1.3792773485183716 (1.3707172735170885) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.488954544067383 (31.406991265036844) + | > grad_norm_1: tensor(249.1201, device='cuda:0') (tensor(134.7266, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.8126 (0.7885782447728243) + | > loader_time: 0.0102 (0.009339522231708874) + + + --> TIME: 2024-06-25 05:26:50 -- STEP: 69/591 -- GLOBAL_STEP: 9525 + | > loss_disc: 2.84120774269104 (2.739324984343156) + | > loss_disc_real_0: 0.1719631552696228 (0.2176690525982691) + | > loss_disc_real_1: 0.22263266146183014 (0.2293268794166869) + | > loss_disc_real_2: 0.2556011974811554 (0.23368434845537378) + | > loss_disc_real_3: 0.23580101132392883 (0.23614743287148682) + | > loss_disc_real_4: 0.25251081585884094 (0.23695326348145804) + | > loss_disc_real_5: 0.2707566022872925 (0.2389123241106669) + | > loss_0: 2.84120774269104 (2.739324984343156) + | > grad_norm_0: tensor(12.5152, device='cuda:0') (tensor(11.9067, device='cuda:0')) + | > loss_gen: 1.9262369871139526 (1.8813545997591987) + | > loss_kl: 2.4210543632507324 (2.584516058797422) + | > loss_feat: 3.0162270069122314 (3.162060844725457) + | > loss_mel: 22.485559463500977 (22.350178151890848) + | > loss_duration: 1.3837940692901611 (1.3781714232071587) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.232872009277344 (31.356281114661176) + | > grad_norm_1: tensor(218.9503, device='cuda:0') (tensor(149.8418, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.8237 (0.7988726194354071) + | > loader_time: 0.0104 (0.009801263394563095) + + + --> TIME: 2024-06-25 05:27:11 -- STEP: 94/591 -- GLOBAL_STEP: 9550 + | > loss_disc: 2.7973244190216064 (2.741398222902989) + | > loss_disc_real_0: 0.2791767716407776 (0.21799511628899174) + | > loss_disc_real_1: 0.19198141992092133 (0.22933709621429443) + | > loss_disc_real_2: 0.24084603786468506 (0.23477339237294298) + | > loss_disc_real_3: 0.24835732579231262 (0.23582818707887163) + | > loss_disc_real_4: 0.21254894137382507 (0.23612623677608816) + | > loss_disc_real_5: 0.22175000607967377 (0.2385954111814499) + | > loss_0: 2.7973244190216064 (2.741398222902989) + | > grad_norm_0: tensor(11.7136, device='cuda:0') (tensor(11.9535, device='cuda:0')) + | > loss_gen: 1.7575050592422485 (1.8752256872806143) + | > loss_kl: 2.495713472366333 (2.5724203180759515) + | > loss_feat: 2.979863405227661 (3.140838341510042) + | > loss_mel: 22.09625244140625 (22.373335006389212) + | > loss_duration: 1.4312090873718262 (1.3866566698601912) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.760543823242188 (31.348476024384194) + | > grad_norm_1: tensor(109.8411, device='cuda:0') (tensor(158.1498, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.8376 (0.8071376516463908) + | > loader_time: 0.0108 (0.010271734379707497) + + + --> TIME: 2024-06-25 05:27:33 -- STEP: 119/591 -- GLOBAL_STEP: 9575 + | > loss_disc: 2.7274811267852783 (2.7390758149764127) + | > loss_disc_real_0: 0.17804931104183197 (0.2171567275243647) + | > loss_disc_real_1: 0.20221669971942902 (0.22883421570813955) + | > loss_disc_real_2: 0.23142457008361816 (0.23438287595240007) + | > loss_disc_real_3: 0.2262962907552719 (0.23579313887768433) + | > loss_disc_real_4: 0.22554300725460052 (0.2359287423246047) + | > loss_disc_real_5: 0.24046136438846588 (0.23869940475756382) + | > loss_0: 2.7274811267852783 (2.7390758149764127) + | > grad_norm_0: tensor(10.7381, device='cuda:0') (tensor(11.3700, device='cuda:0')) + | > loss_gen: 1.9328362941741943 (1.8745715047131066) + | > loss_kl: 2.553774356842041 (2.554074742213017) + | > loss_feat: 3.3805503845214844 (3.140277313585041) + | > loss_mel: 22.618518829345703 (22.347391593356093) + | > loss_duration: 1.3728506565093994 (1.3883432470449886) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.858530044555664 (31.304658344813756) + | > grad_norm_1: tensor(95.6885, device='cuda:0') (tensor(149.7045, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.8399 (0.8132628813511183) + | > loader_time: 0.0121 (0.010568514591505545) + + + --> TIME: 2024-06-25 05:27:54 -- STEP: 144/591 -- GLOBAL_STEP: 9600 + | > loss_disc: 2.742354154586792 (2.7422912783092936) + | > loss_disc_real_0: 0.19383910298347473 (0.2178404493153923) + | > loss_disc_real_1: 0.23173417150974274 (0.22912033585210642) + | > loss_disc_real_2: 0.23613432049751282 (0.23503840073115295) + | > loss_disc_real_3: 0.24260826408863068 (0.2357993292518788) + | > loss_disc_real_4: 0.24301224946975708 (0.2361787733518415) + | > loss_disc_real_5: 0.2194615602493286 (0.2390177356493142) + | > loss_0: 2.742354154586792 (2.7422912783092936) + | > grad_norm_0: tensor(5.7387, device='cuda:0') (tensor(11.3623, device='cuda:0')) + | > loss_gen: 1.8070716857910156 (1.8721982911229134) + | > loss_kl: 2.5415704250335693 (2.5454596562518015) + | > loss_feat: 2.8806216716766357 (3.1305437468820148) + | > loss_mel: 22.223495483398438 (22.338108023007713) + | > loss_duration: 1.3697097301483154 (1.3888618283801613) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.822467803955078 (31.275171491834854) + | > grad_norm_1: tensor(110.6407, device='cuda:0') (tensor(142.4143, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.8564 (0.8198588887850443) + | > loader_time: 0.0116 (0.010754633280966014) + + + --> TIME: 2024-06-25 05:28:16 -- STEP: 169/591 -- GLOBAL_STEP: 9625 + | > loss_disc: 2.78385591506958 (2.7424633658143898) + | > loss_disc_real_0: 0.3403159976005554 (0.21822435351518485) + | > loss_disc_real_1: 0.23111596703529358 (0.2291770462806408) + | > loss_disc_real_2: 0.21762588620185852 (0.23517132122841108) + | > loss_disc_real_3: 0.27964484691619873 (0.23589915460383398) + | > loss_disc_real_4: 0.241853266954422 (0.2360014849336895) + | > loss_disc_real_5: 0.2317916750907898 (0.2386086906730776) + | > loss_0: 2.78385591506958 (2.7424633658143898) + | > grad_norm_0: tensor(20.1729, device='cuda:0') (tensor(11.1535, device='cuda:0')) + | > loss_gen: 1.7549560070037842 (1.871863961925168) + | > loss_kl: 2.54923415184021 (2.5401473412146935) + | > loss_feat: 3.2389121055603027 (3.1240880785609133) + | > loss_mel: 22.035057067871094 (22.3353683765118) + | > loss_duration: 1.368531346321106 (1.3870441716098225) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.946691513061523 (31.25851187621348) + | > grad_norm_1: tensor(70.1048, device='cuda:0') (tensor(138.1808, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.8497 (0.8262399921755819) + | > loader_time: 0.0117 (0.010981016610501075) + + + --> TIME: 2024-06-25 05:28:39 -- STEP: 194/591 -- GLOBAL_STEP: 9650 + | > loss_disc: 2.7243118286132812 (2.743273667453492) + | > loss_disc_real_0: 0.270799458026886 (0.2185256852163482) + | > loss_disc_real_1: 0.1810445338487625 (0.22894563518234134) + | > loss_disc_real_2: 0.24803313612937927 (0.23552368312459632) + | > loss_disc_real_3: 0.22537145018577576 (0.23560130066171134) + | > loss_disc_real_4: 0.24663175642490387 (0.23579291738185687) + | > loss_disc_real_5: 0.2594253420829773 (0.23862212619830653) + | > loss_0: 2.7243118286132812 (2.743273667453492) + | > grad_norm_0: tensor(10.7406, device='cuda:0') (tensor(11.4032, device='cuda:0')) + | > loss_gen: 1.8627066612243652 (1.872007448648669) + | > loss_kl: 2.3891146183013916 (2.5313644273993887) + | > loss_feat: 3.054441213607788 (3.1176554050642187) + | > loss_mel: 22.349363327026367 (22.329159579326202) + | > loss_duration: 1.3986163139343262 (1.3858737453971939) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.054243087768555 (31.236060516121462) + | > grad_norm_1: tensor(249.6996, device='cuda:0') (tensor(140.5758, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.8659 (0.8315095090374505) + | > loader_time: 0.0116 (0.011163731211239526) + + + --> TIME: 2024-06-25 05:29:01 -- STEP: 219/591 -- GLOBAL_STEP: 9675 + | > loss_disc: 2.7210376262664795 (2.744167935358336) + | > loss_disc_real_0: 0.30613604187965393 (0.21881742584923086) + | > loss_disc_real_1: 0.25883808732032776 (0.22923262451337353) + | > loss_disc_real_2: 0.22469443082809448 (0.2354536466130383) + | > loss_disc_real_3: 0.3147040009498596 (0.2358021319729008) + | > loss_disc_real_4: 0.2168474793434143 (0.23555941954595314) + | > loss_disc_real_5: 0.2175913155078888 (0.23853225556954946) + | > loss_0: 2.7210376262664795 (2.744167935358336) + | > grad_norm_0: tensor(20.9149, device='cuda:0') (tensor(11.3410, device='cuda:0')) + | > loss_gen: 1.824459195137024 (1.8711968056143147) + | > loss_kl: 2.353705644607544 (2.5294443537655478) + | > loss_feat: 3.143040418624878 (3.110080106073318) + | > loss_mel: 22.41246795654297 (22.330662548814193) + | > loss_duration: 1.3760253190994263 (1.385112014535355) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.109699249267578 (31.226495751507205) + | > grad_norm_1: tensor(49.0803, device='cuda:0') (tensor(141.7999, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.8816 (0.8378819864090175) + | > loader_time: 0.0139 (0.011412118667881234) + + + --> TIME: 2024-06-25 05:29:24 -- STEP: 244/591 -- GLOBAL_STEP: 9700 + | > loss_disc: 2.7737700939178467 (2.7443154639885083) + | > loss_disc_real_0: 0.18429580330848694 (0.21883158336897365) + | > loss_disc_real_1: 0.22682613134384155 (0.2289478345850452) + | > loss_disc_real_2: 0.24266262352466583 (0.23555298461044422) + | > loss_disc_real_3: 0.29186075925827026 (0.2357718527439188) + | > loss_disc_real_4: 0.22603346407413483 (0.23561748068352215) + | > loss_disc_real_5: 0.26354849338531494 (0.23873203430996567) + | > loss_0: 2.7737700939178467 (2.7443154639885083) + | > grad_norm_0: tensor(16.9167, device='cuda:0') (tensor(11.5525, device='cuda:0')) + | > loss_gen: 2.026111125946045 (1.8720454813026992) + | > loss_kl: 2.589182138442993 (2.526466057925927) + | > loss_feat: 2.862550973892212 (3.109551041829781) + | > loss_mel: 22.657917022705078 (22.345616371905223) + | > loss_duration: 1.3673909902572632 (1.3853579222178847) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.50315284729004 (31.23903678675167) + | > grad_norm_1: tensor(204.7807, device='cuda:0') (tensor(136.4727, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.8941 (0.8429665956340853) + | > loader_time: 0.0141 (0.011654536255070416) + + + --> TIME: 2024-06-25 05:29:47 -- STEP: 269/591 -- GLOBAL_STEP: 9725 + | > loss_disc: 2.7794198989868164 (2.7433956192328592) + | > loss_disc_real_0: 0.27332931756973267 (0.21887435046713585) + | > loss_disc_real_1: 0.27000483870506287 (0.22915828266773083) + | > loss_disc_real_2: 0.2547702193260193 (0.23543993491665582) + | > loss_disc_real_3: 0.2290492057800293 (0.23578489208974804) + | > loss_disc_real_4: 0.22538205981254578 (0.2350255813740443) + | > loss_disc_real_5: 0.22742125391960144 (0.23831183643367654) + | > loss_0: 2.7794198989868164 (2.7433956192328592) + | > grad_norm_0: tensor(9.9606, device='cuda:0') (tensor(11.4076, device='cuda:0')) + | > loss_gen: 1.9333961009979248 (1.8723487752077748) + | > loss_kl: 2.650667667388916 (2.5231266695327466) + | > loss_feat: 3.045785903930664 (3.1056756804866854) + | > loss_mel: 22.25295066833496 (22.314068237645035) + | > loss_duration: 1.4119067192077637 (1.3852832290762862) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.294706344604492 (31.200502516168644) + | > grad_norm_1: tensor(172.2041, device='cuda:0') (tensor(136.3276, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.8984 (0.8483035537833175) + | > loader_time: 0.0155 (0.011943705463054895) + + + --> TIME: 2024-06-25 05:30:10 -- STEP: 294/591 -- GLOBAL_STEP: 9750 + | > loss_disc: 2.7256815433502197 (2.742647342941389) + | > loss_disc_real_0: 0.3015463352203369 (0.2189174528126002) + | > loss_disc_real_1: 0.18430748581886292 (0.2288757173382506) + | > loss_disc_real_2: 0.1956375390291214 (0.23519557450904327) + | > loss_disc_real_3: 0.25826284289360046 (0.23576109025145875) + | > loss_disc_real_4: 0.21887369453907013 (0.2349774367651161) + | > loss_disc_real_5: 0.2289769947528839 (0.23822575001692287) + | > loss_0: 2.7256815433502197 (2.742647342941389) + | > grad_norm_0: tensor(16.8453, device='cuda:0') (tensor(11.3662, device='cuda:0')) + | > loss_gen: 1.9383926391601562 (1.8723026595148107) + | > loss_kl: 2.5970458984375 (2.521251650894579) + | > loss_feat: 3.0482208728790283 (3.1026743861282764) + | > loss_mel: 21.729907989501953 (22.287450725529474) + | > loss_duration: 1.363793134689331 (1.3858014385716442) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.67736053466797 (31.169480790897293) + | > grad_norm_1: tensor(129.1731, device='cuda:0') (tensor(138.0061, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.9217 (0.8534596169075997) + | > loader_time: 0.0168 (0.01228323599108222) + + + --> TIME: 2024-06-25 05:30:33 -- STEP: 319/591 -- GLOBAL_STEP: 9775 + | > loss_disc: 2.763897657394409 (2.7440976827495906) + | > loss_disc_real_0: 0.1388741433620453 (0.21907853795256352) + | > loss_disc_real_1: 0.19814243912696838 (0.2291255574147903) + | > loss_disc_real_2: 0.20840001106262207 (0.23531207839336515) + | > loss_disc_real_3: 0.18224728107452393 (0.23570951996925857) + | > loss_disc_real_4: 0.1992247849702835 (0.2351296415616726) + | > loss_disc_real_5: 0.20074470341205597 (0.23825550579165217) + | > loss_0: 2.763897657394409 (2.7440976827495906) + | > grad_norm_0: tensor(18.5717, device='cuda:0') (tensor(11.7310, device='cuda:0')) + | > loss_gen: 1.8675333261489868 (1.8715580604666833) + | > loss_kl: 2.416677236557007 (2.5192002278510297) + | > loss_feat: 3.003110647201538 (3.0987349595395743) + | > loss_mel: 22.952367782592773 (22.284751383862165) + | > loss_duration: 1.3774176836013794 (1.3862713855635784) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.617107391357422 (31.16051595562304) + | > grad_norm_1: tensor(101.1258, device='cuda:0') (tensor(136.6003, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.9446 (0.8585740040088522) + | > loader_time: 0.0139 (0.012514263111222121) + + + --> TIME: 2024-06-25 05:30:57 -- STEP: 344/591 -- GLOBAL_STEP: 9800 + | > loss_disc: 2.7458319664001465 (2.743428748014362) + | > loss_disc_real_0: 0.18419256806373596 (0.2190172213541213) + | > loss_disc_real_1: 0.20961444079875946 (0.22908069409949836) + | > loss_disc_real_2: 0.21291321516036987 (0.23531139295461567) + | > loss_disc_real_3: 0.22282283008098602 (0.2357923239036355) + | > loss_disc_real_4: 0.2284485250711441 (0.23511102544360382) + | > loss_disc_real_5: 0.2623031735420227 (0.2382744426817395) + | > loss_0: 2.7458319664001465 (2.743428748014362) + | > grad_norm_0: tensor(8.9123, device='cuda:0') (tensor(11.6035, device='cuda:0')) + | > loss_gen: 1.6899465322494507 (1.8703653691119932) + | > loss_kl: 2.502716302871704 (2.51701241870259) + | > loss_feat: 2.968172788619995 (3.09452633732973) + | > loss_mel: 22.121795654296875 (22.274311991625062) + | > loss_duration: 1.3926098346710205 (1.3872273942065798) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.675243377685547 (31.143443440282066) + | > grad_norm_1: tensor(75.0668, device='cuda:0') (tensor(133.0476, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.9306 (0.8639440786006839) + | > loader_time: 0.0183 (0.012755476458128103) + + + --> TIME: 2024-06-25 05:31:22 -- STEP: 369/591 -- GLOBAL_STEP: 9825 + | > loss_disc: 2.8024938106536865 (2.744234450787388) + | > loss_disc_real_0: 0.29373785853385925 (0.21895532164631812) + | > loss_disc_real_1: 0.26450735330581665 (0.2293011350279578) + | > loss_disc_real_2: 0.23833203315734863 (0.2354616285743429) + | > loss_disc_real_3: 0.22557951509952545 (0.23583173392426354) + | > loss_disc_real_4: 0.26098963618278503 (0.23524843128069953) + | > loss_disc_real_5: 0.24640224874019623 (0.238287385401687) + | > loss_0: 2.8024938106536865 (2.744234450787388) + | > grad_norm_0: tensor(14.4713, device='cuda:0') (tensor(11.5080, device='cuda:0')) + | > loss_gen: 1.7234294414520264 (1.8694475598451574) + | > loss_kl: 2.450648069381714 (2.5132736436073686) + | > loss_feat: 2.7530603408813477 (3.0863536580145183) + | > loss_mel: 21.64000129699707 (22.265354838797716) + | > loss_duration: 1.4153234958648682 (1.3891234336506706) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.98246192932129 (31.12355306671887) + | > grad_norm_1: tensor(93.2936, device='cuda:0') (tensor(132.2635, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.9427 (0.8695187807729251) + | > loader_time: 0.0178 (0.013012126855410852) + + + --> TIME: 2024-06-25 05:31:46 -- STEP: 394/591 -- GLOBAL_STEP: 9850 + | > loss_disc: 2.8315536975860596 (2.7450888447350064) + | > loss_disc_real_0: 0.37349480390548706 (0.21907402317838615) + | > loss_disc_real_1: 0.2690090239048004 (0.22919179820621075) + | > loss_disc_real_2: 0.26108816266059875 (0.23552796550964947) + | > loss_disc_real_3: 0.25078022480010986 (0.23585429004757538) + | > loss_disc_real_4: 0.25758036971092224 (0.23528409393759547) + | > loss_disc_real_5: 0.2748066782951355 (0.23845740884239905) + | > loss_0: 2.8315536975860596 (2.7450888447350064) + | > grad_norm_0: tensor(28.0687, device='cuda:0') (tensor(11.5319, device='cuda:0')) + | > loss_gen: 1.7771166563034058 (1.869163118340643) + | > loss_kl: 2.531386613845825 (2.511439257466852) + | > loss_feat: 2.80067777633667 (3.080811141106077) + | > loss_mel: 22.28710174560547 (22.255303218279998) + | > loss_duration: 1.3690857887268066 (1.3899620983201235) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.765369415283203 (31.106678778750037) + | > grad_norm_1: tensor(148.5592, device='cuda:0') (tensor(135.2565, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.9633 (0.8763087969746082) + | > loader_time: 0.0172 (0.01326205524696311) + + + --> TIME: 2024-06-25 05:32:12 -- STEP: 419/591 -- GLOBAL_STEP: 9875 + | > loss_disc: 2.723367214202881 (2.7461596661934147) + | > loss_disc_real_0: 0.1620350182056427 (0.2189995151431008) + | > loss_disc_real_1: 0.2255740910768509 (0.2294275538377944) + | > loss_disc_real_2: 0.22878409922122955 (0.23552165936171865) + | > loss_disc_real_3: 0.2265934944152832 (0.2358946054106965) + | > loss_disc_real_4: 0.2518610954284668 (0.23547663665898944) + | > loss_disc_real_5: 0.26529639959335327 (0.2385170896113061) + | > loss_0: 2.723367214202881 (2.7461596661934147) + | > grad_norm_0: tensor(16.2475, device='cuda:0') (tensor(11.5781, device='cuda:0')) + | > loss_gen: 1.8334972858428955 (1.867659879755007) + | > loss_kl: 2.417098045349121 (2.508540187075483) + | > loss_feat: 2.706416606903076 (3.0711087571692626) + | > loss_mel: 22.35635757446289 (22.23561122138631) + | > loss_duration: 1.4365005493164062 (1.3906732060744256) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.74987030029297 (31.073593176065597) + | > grad_norm_1: tensor(78.1724, device='cuda:0') (tensor(134.8007, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 0.9595 (0.8830262425406736) + | > loader_time: 0.0162 (0.01352468481496296) + + + --> TIME: 2024-06-25 05:32:37 -- STEP: 444/591 -- GLOBAL_STEP: 9900 + | > loss_disc: 2.6754541397094727 (2.746729887283601) + | > loss_disc_real_0: 0.15473398566246033 (0.21911112175465697) + | > loss_disc_real_1: 0.19604766368865967 (0.22925322194088688) + | > loss_disc_real_2: 0.2146451622247696 (0.23548539751419076) + | > loss_disc_real_3: 0.19267112016677856 (0.23602555339803566) + | > loss_disc_real_4: 0.23521238565444946 (0.235343420257171) + | > loss_disc_real_5: 0.26200827956199646 (0.23853465840891674) + | > loss_0: 2.6754541397094727 (2.746729887283601) + | > grad_norm_0: tensor(9.9557, device='cuda:0') (tensor(11.6134, device='cuda:0')) + | > loss_gen: 1.903376817703247 (1.8659034406279664) + | > loss_kl: 2.510270595550537 (2.5050144131119167) + | > loss_feat: 3.0775346755981445 (3.0623339720674454) + | > loss_mel: 21.716440200805664 (22.213539780797184) + | > loss_duration: 1.4136829376220703 (1.391518274943034) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.621305465698242 (31.038309827581184) + | > grad_norm_1: tensor(174.3468, device='cuda:0') (tensor(133.3803, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 1.005 (0.8898299851933041) + | > loader_time: 0.0224 (0.013790387291092053) + + + --> TIME: 2024-06-25 05:33:04 -- STEP: 469/591 -- GLOBAL_STEP: 9925 + | > loss_disc: 2.755464553833008 (2.7458836773056983) + | > loss_disc_real_0: 0.2536923885345459 (0.21896902092102996) + | > loss_disc_real_1: 0.2226671427488327 (0.22933386282117635) + | > loss_disc_real_2: 0.24999459087848663 (0.23534834124386184) + | > loss_disc_real_3: 0.22871752083301544 (0.2359879505214915) + | > loss_disc_real_4: 0.249959796667099 (0.23525967434652326) + | > loss_disc_real_5: 0.2667984664440155 (0.2385953442374272) + | > loss_0: 2.755464553833008 (2.7458836773056983) + | > grad_norm_0: tensor(7.8502, device='cuda:0') (tensor(11.5455, device='cuda:0')) + | > loss_gen: 1.868130087852478 (1.865810845706509) + | > loss_kl: 2.3861186504364014 (2.502645073160686) + | > loss_feat: 2.7352232933044434 (3.058241116212628) + | > loss_mel: 21.17906379699707 (22.19455929656527) + | > loss_duration: 1.402578353881836 (1.3926884190105935) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.57111358642578 (31.01394469905764) + | > grad_norm_1: tensor(158.9649, device='cuda:0') (tensor(133.3224, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 1.1121 (0.8972551675239352) + | > loader_time: 0.0183 (0.014024783299167523) + + + --> TIME: 2024-06-25 05:33:31 -- STEP: 494/591 -- GLOBAL_STEP: 9950 + | > loss_disc: 2.7822506427764893 (2.746516701663555) + | > loss_disc_real_0: 0.1505257934331894 (0.2188375701728137) + | > loss_disc_real_1: 0.16098658740520477 (0.22943970042201672) + | > loss_disc_real_2: 0.22520342469215393 (0.23552040429974375) + | > loss_disc_real_3: 0.23072564601898193 (0.23596646739283073) + | > loss_disc_real_4: 0.21866543591022491 (0.23534606089476148) + | > loss_disc_real_5: 0.24776864051818848 (0.23853472629298084) + | > loss_0: 2.7822506427764893 (2.746516701663555) + | > grad_norm_0: tensor(19.8570, device='cuda:0') (tensor(11.6247, device='cuda:0')) + | > loss_gen: 1.8543199300765991 (1.8660160932946306) + | > loss_kl: 2.43894624710083 (2.5008875315005943) + | > loss_feat: 3.050612449645996 (3.054619865378869) + | > loss_mel: 21.897232055664062 (22.188286781311035) + | > loss_duration: 1.3974394798278809 (1.394117063597629) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.638551712036133 (31.003927288750404) + | > grad_norm_1: tensor(225.9370, device='cuda:0') (tensor(135.3768, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 1.1501 (0.9053479148308757) + | > loader_time: 0.0162 (0.014280423944295656) + + + --> TIME: 2024-06-25 05:33:59 -- STEP: 519/591 -- GLOBAL_STEP: 9975 + | > loss_disc: 2.78037691116333 (2.7471569264325573) + | > loss_disc_real_0: 0.23698797821998596 (0.2187808408098643) + | > loss_disc_real_1: 0.22233571112155914 (0.22976774353397605) + | > loss_disc_real_2: 0.2721311151981354 (0.2356102075248325) + | > loss_disc_real_3: 0.24264298379421234 (0.2359597431729065) + | > loss_disc_real_4: 0.24041345715522766 (0.2354581338534695) + | > loss_disc_real_5: 0.26768946647644043 (0.2385437026304087) + | > loss_0: 2.78037691116333 (2.7471569264325573) + | > grad_norm_0: tensor(5.4406, device='cuda:0') (tensor(11.5458, device='cuda:0')) + | > loss_gen: 1.7755318880081177 (1.8658660574455486) + | > loss_kl: 2.3469929695129395 (2.498273267451966) + | > loss_feat: 2.6894659996032715 (3.04904726934341) + | > loss_mel: 20.691020965576172 (22.176781424667563) + | > loss_duration: 1.4072682857513428 (1.3955606529937774) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 28.910280227661133 (30.985528629868934) + | > grad_norm_1: tensor(159.4179, device='cuda:0') (tensor(138.0255, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 1.1218 (0.9145436374208601) + | > loader_time: 0.0198 (0.01452305229642717) + + + --> TIME: 2024-06-25 05:34:28 -- STEP: 544/591 -- GLOBAL_STEP: 10000 + | > loss_disc: 2.702601671218872 (2.7477407319580816) + | > loss_disc_real_0: 0.2381991744041443 (0.218921075684621) + | > loss_disc_real_1: 0.21185947954654694 (0.22983652391635323) + | > loss_disc_real_2: 0.24037323892116547 (0.23574491809396184) + | > loss_disc_real_3: 0.1878831535577774 (0.23586770766140777) + | > loss_disc_real_4: 0.2500035762786865 (0.23556073409888675) + | > loss_disc_real_5: 0.25784537196159363 (0.23839240882764845) + | > loss_0: 2.702601671218872 (2.7477407319580816) + | > grad_norm_0: tensor(7.8528, device='cuda:0') (tensor(11.6066, device='cuda:0')) + | > loss_gen: 1.9655473232269287 (1.865968899472672) + | > loss_kl: 2.4885659217834473 (2.4957763175753964) + | > loss_feat: 3.150813341140747 (3.0433790249859576) + | > loss_mel: 22.02967643737793 (22.16126355353524) + | > loss_duration: 1.4216848611831665 (1.396981837118373) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.05628776550293 (30.963369593900794) + | > grad_norm_1: tensor(111.7236, device='cuda:0') (tensor(136.5478, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 1.2353 (0.9254740910495028) + | > loader_time: 0.0278 (0.014832778450320746) + + + > CHECKPOINT : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/checkpoint_10000.pth + + --> TIME: 2024-06-25 05:35:02 -- STEP: 569/591 -- GLOBAL_STEP: 10025 + | > loss_disc: 2.8137598037719727 (2.748912365449125) + | > loss_disc_real_0: 0.10973100364208221 (0.21900758085697095) + | > loss_disc_real_1: 0.26161989569664 (0.2299550499616482) + | > loss_disc_real_2: 0.2667251527309418 (0.2358787510485138) + | > loss_disc_real_3: 0.26897504925727844 (0.23599965359707914) + | > loss_disc_real_4: 0.22345595061779022 (0.23557010981760043) + | > loss_disc_real_5: 0.24595634639263153 (0.2384241795288448) + | > loss_0: 2.8137598037719727 (2.748912365449125) + | > grad_norm_0: tensor(26.9274, device='cuda:0') (tensor(11.6016, device='cuda:0')) + | > loss_gen: 1.9719643592834473 (1.8649979926161275) + | > loss_kl: 2.3604767322540283 (2.4922540355440996) + | > loss_feat: 2.8981852531433105 (3.0354646115395205) + | > loss_mel: 21.957584381103516 (22.142540319942217) + | > loss_duration: 1.451610803604126 (1.398491121344072) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.639822006225586 (30.93374803740865) + | > grad_norm_1: tensor(283.6968, device='cuda:0') (tensor(136.5352, device='cuda:0')) + | > current_lr_0: 0.00019960037478133875 + | > current_lr_1: 0.00019960037478133875 + | > step_time: 1.2419 (0.9382489108871701) + | > loader_time: 0.0218 (0.015176043569098874) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036005923088560715 (-1.3199258357920324e-05) + | > avg_loss_disc: 2.769175996171667 (-0.008622828950274108) + | > avg_loss_disc_real_0: 0.29047953543510835 (+0.10198126923530651) + | > avg_loss_disc_real_1: 0.2012914235287524 (-0.03786264074609638) + | > avg_loss_disc_real_2: 0.2625349316191166 (-0.03280464258599791) + | > avg_loss_disc_real_3: 0.19637675646771777 (-0.034281334661422885) + | > avg_loss_disc_real_4: 0.2094730037323972 (-0.05028879959532556) + | > avg_loss_disc_real_5: 0.22074585804279814 (-0.05079783435831678) + | > avg_loss_0: 2.769175996171667 (-0.008622828950274108) + | > avg_loss_gen: 1.8140518639950043 (-0.061313154849600204) + | > avg_loss_kl: 2.551311366101529 (+0.09844091598023796) + | > avg_loss_feat: 3.0148672296645795 (+0.1478023275415956) + | > avg_loss_mel: 22.235454194089193 (+0.11340725675541563) + | > avg_loss_duration: 1.3883707599436983 (-0.00947998432402919) + | > avg_loss_1: 31.00405538843033 (+0.2888574397310286) + + + > EPOCH: 18/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 05:35:37)  + + --> TIME: 2024-06-25 05:35:41 -- STEP: 3/591 -- GLOBAL_STEP: 10050 + | > loss_disc: 2.758439779281616 (2.742318948109945) + | > loss_disc_real_0: 0.26335930824279785 (0.21708117425441742) + | > loss_disc_real_1: 0.23015666007995605 (0.24646753569444022) + | > loss_disc_real_2: 0.25795552134513855 (0.23540269831816354) + | > loss_disc_real_3: 0.22273731231689453 (0.24429654578367868) + | > loss_disc_real_4: 0.2273411899805069 (0.23929535845915476) + | > loss_disc_real_5: 0.30044567584991455 (0.25597022970517475) + | > loss_0: 2.758439779281616 (2.742318948109945) + | > grad_norm_0: tensor(12.4166, device='cuda:0') (tensor(8.3492, device='cuda:0')) + | > loss_gen: 1.757406234741211 (1.9299110571543376) + | > loss_kl: 2.781101942062378 (2.6964524586995444) + | > loss_feat: 3.159750461578369 (3.3277101516723633) + | > loss_mel: 22.32107162475586 (22.60848108927409) + | > loss_duration: 1.3524717092514038 (1.3722588618596394) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.371803283691406 (31.934813181559246) + | > grad_norm_1: tensor(196.1750, device='cuda:0') (tensor(148.1299, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.7566 (0.7689518133799235) + | > loader_time: 0.007 (0.007650931676228841) + + + --> TIME: 2024-06-25 05:36:01 -- STEP: 28/591 -- GLOBAL_STEP: 10075 + | > loss_disc: 2.7945709228515625 (2.7582535488264903) + | > loss_disc_real_0: 0.33205360174179077 (0.22136255194033896) + | > loss_disc_real_1: 0.24900493025779724 (0.23089952554021562) + | > loss_disc_real_2: 0.2937769293785095 (0.23756220936775208) + | > loss_disc_real_3: 0.2946435809135437 (0.23978003593427794) + | > loss_disc_real_4: 0.2385294884443283 (0.23756182406629836) + | > loss_disc_real_5: 0.2743544280529022 (0.23984203434416226) + | > loss_0: 2.7945709228515625 (2.7582535488264903) + | > grad_norm_0: tensor(19.6398, device='cuda:0') (tensor(13.9559, device='cuda:0')) + | > loss_gen: 1.871096134185791 (1.8840475721018655) + | > loss_kl: 2.6577067375183105 (2.662177690437862) + | > loss_feat: 3.0307281017303467 (3.1432727915900096) + | > loss_mel: 22.635494232177734 (22.252574239458358) + | > loss_duration: 1.3796923160552979 (1.354537912777492) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.574716567993164 (31.29661008289882) + | > grad_norm_1: tensor(52.2405, device='cuda:0') (tensor(137.0228, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.7888 (0.7802679112979344) + | > loader_time: 0.0091 (0.00889815602983747) + + + --> TIME: 2024-06-25 05:36:22 -- STEP: 53/591 -- GLOBAL_STEP: 10100 + | > loss_disc: 2.7560720443725586 (2.747695464008259) + | > loss_disc_real_0: 0.34312695264816284 (0.2213596927950967) + | > loss_disc_real_1: 0.18951301276683807 (0.2276049205717051) + | > loss_disc_real_2: 0.23775003850460052 (0.23420007695566933) + | > loss_disc_real_3: 0.21065379679203033 (0.2361631562125008) + | > loss_disc_real_4: 0.26914963126182556 (0.23879300929465383) + | > loss_disc_real_5: 0.2489359974861145 (0.23954691302101566) + | > loss_0: 2.7560720443725586 (2.747695464008259) + | > grad_norm_0: tensor(23.2654, device='cuda:0') (tensor(12.3410, device='cuda:0')) + | > loss_gen: 1.9668786525726318 (1.8828198504897784) + | > loss_kl: 2.601126194000244 (2.609390618666162) + | > loss_feat: 3.2371926307678223 (3.1633784770965576) + | > loss_mel: 22.07925796508789 (22.23805809020996) + | > loss_duration: 1.3679131269454956 (1.3697347303606429) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.25236701965332 (31.26338167010613) + | > grad_norm_1: tensor(140.1459, device='cuda:0') (tensor(134.3957, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.8336 (0.797700342142357) + | > loader_time: 0.01 (0.009276484543422482) + + + --> TIME: 2024-06-25 05:36:43 -- STEP: 78/591 -- GLOBAL_STEP: 10125 + | > loss_disc: 2.714250326156616 (2.7517237449303655) + | > loss_disc_real_0: 0.1259588897228241 (0.22099570146737954) + | > loss_disc_real_1: 0.24295884370803833 (0.2292312482992808) + | > loss_disc_real_2: 0.23561425507068634 (0.2340060627231231) + | > loss_disc_real_3: 0.24750164151191711 (0.2360935885554705) + | > loss_disc_real_4: 0.23630477488040924 (0.2376960989756462) + | > loss_disc_real_5: 0.2334335595369339 (0.23997178272559092) + | > loss_0: 2.714250326156616 (2.7517237449303655) + | > grad_norm_0: tensor(21.7443, device='cuda:0') (tensor(12.3886, device='cuda:0')) + | > loss_gen: 2.0236785411834717 (1.877044058763064) + | > loss_kl: 2.3703343868255615 (2.5702814658482858) + | > loss_feat: 3.1687405109405518 (3.1420588890711465) + | > loss_mel: 21.90435218811035 (22.222030248397438) + | > loss_duration: 1.4188916683197021 (1.3766621626340425) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.885997772216797 (31.188076801789112) + | > grad_norm_1: tensor(111.0139, device='cuda:0') (tensor(141.2028, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.8227 (0.8044500014720817) + | > loader_time: 0.0101 (0.009649600738134142) + + + --> TIME: 2024-06-25 05:37:04 -- STEP: 103/591 -- GLOBAL_STEP: 10150 + | > loss_disc: 2.68556547164917 (2.7485846500952262) + | > loss_disc_real_0: 0.217091903090477 (0.22155287717152566) + | > loss_disc_real_1: 0.2625645101070404 (0.22933906852041633) + | > loss_disc_real_2: 0.23158803582191467 (0.2341759100701045) + | > loss_disc_real_3: 0.26363345980644226 (0.23592948609764136) + | > loss_disc_real_4: 0.2639181315898895 (0.23675810755456536) + | > loss_disc_real_5: 0.23216885328292847 (0.23972996360468632) + | > loss_0: 2.68556547164917 (2.7485846500952262) + | > grad_norm_0: tensor(4.5905, device='cuda:0') (tensor(12.1622, device='cuda:0')) + | > loss_gen: 1.7853925228118896 (1.8762218547098846) + | > loss_kl: 2.594975471496582 (2.5537952844378995) + | > loss_feat: 3.3490564823150635 (3.1521983216109786) + | > loss_mel: 22.15831184387207 (22.17762847085601) + | > loss_duration: 1.3832066059112549 (1.3831508842486777) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.270944595336914 (31.142994871417297) + | > grad_norm_1: tensor(101.9218, device='cuda:0') (tensor(135.4884, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.8484 (0.8115941876346623) + | > loader_time: 0.0122 (0.0100400957089026) + + + --> TIME: 2024-06-25 05:37:26 -- STEP: 128/591 -- GLOBAL_STEP: 10175 + | > loss_disc: 2.739751100540161 (2.748080546036362) + | > loss_disc_real_0: 0.2500608265399933 (0.22062163043301553) + | > loss_disc_real_1: 0.23888936638832092 (0.2294023836730048) + | > loss_disc_real_2: 0.2098923921585083 (0.23377869906835258) + | > loss_disc_real_3: 0.24064090847969055 (0.2358993204543367) + | > loss_disc_real_4: 0.25487610697746277 (0.2363193065393716) + | > loss_disc_real_5: 0.24593238532543182 (0.23934877628926188) + | > loss_0: 2.739751100540161 (2.748080546036362) + | > grad_norm_0: tensor(8.8075, device='cuda:0') (tensor(11.7289, device='cuda:0')) + | > loss_gen: 1.9604016542434692 (1.8745130486786366) + | > loss_kl: 2.5620429515838623 (2.541546005755661) + | > loss_feat: 3.367798328399658 (3.150639256462455) + | > loss_mel: 22.614423751831055 (22.19252184033394) + | > loss_duration: 1.3766015768051147 (1.3850509477779267) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.881269454956055 (31.144271105527878) + | > grad_norm_1: tensor(237.8088, device='cuda:0') (tensor(140.9097, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.8527 (0.8176977559924123) + | > loader_time: 0.0121 (0.010338269174098969) + + + --> TIME: 2024-06-25 05:37:47 -- STEP: 153/591 -- GLOBAL_STEP: 10200 + | > loss_disc: 2.766554117202759 (2.7475541121040297) + | > loss_disc_real_0: 0.165238156914711 (0.22036895720787297) + | > loss_disc_real_1: 0.23963813483715057 (0.22993006036172514) + | > loss_disc_real_2: 0.24489213526248932 (0.23435142402555428) + | > loss_disc_real_3: 0.18788987398147583 (0.23520474817628176) + | > loss_disc_real_4: 0.237399622797966 (0.23610532147432464) + | > loss_disc_real_5: 0.2298368513584137 (0.23889347667398017) + | > loss_0: 2.766554117202759 (2.7475541121040297) + | > grad_norm_0: tensor(15.5925, device='cuda:0') (tensor(11.5521, device='cuda:0')) + | > loss_gen: 2.0006370544433594 (1.874321456049003) + | > loss_kl: 2.5291786193847656 (2.5384792673821526) + | > loss_feat: 3.0430169105529785 (3.1511650459439147) + | > loss_mel: 21.788944244384766 (22.190193674922774) + | > loss_duration: 1.3647416830062866 (1.3833001929950086) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.726518630981445 (31.13745958664838) + | > grad_norm_1: tensor(41.8902, device='cuda:0') (tensor(139.6991, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.859 (0.823643056395786) + | > loader_time: 0.0111 (0.010615947199802772) + + + --> TIME: 2024-06-25 05:38:09 -- STEP: 178/591 -- GLOBAL_STEP: 10225 + | > loss_disc: 2.6945104598999023 (2.7480319724993767) + | > loss_disc_real_0: 0.18727031350135803 (0.2210944884278801) + | > loss_disc_real_1: 0.23999132215976715 (0.23022145041254138) + | > loss_disc_real_2: 0.22198140621185303 (0.2343558080578118) + | > loss_disc_real_3: 0.264726847410202 (0.23546103815014444) + | > loss_disc_real_4: 0.24531319737434387 (0.23593025830354583) + | > loss_disc_real_5: 0.22583304345607758 (0.2385153892669785) + | > loss_0: 2.6945104598999023 (2.7480319724993767) + | > grad_norm_0: tensor(7.2672, device='cuda:0') (tensor(11.5853, device='cuda:0')) + | > loss_gen: 1.942244052886963 (1.8752205787080058) + | > loss_kl: 2.444730043411255 (2.5309767214100005) + | > loss_feat: 3.416081428527832 (3.153099782011481) + | > loss_mel: 22.44211769104004 (22.18958903966325) + | > loss_duration: 1.3619787693023682 (1.3817574267976742) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.60715103149414 (31.130643501710356) + | > grad_norm_1: tensor(91.0388, device='cuda:0') (tensor(132.5652, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.8735 (0.8291582177194315) + | > loader_time: 0.0117 (0.010832517334584438) + + + --> TIME: 2024-06-25 05:38:32 -- STEP: 203/591 -- GLOBAL_STEP: 10250 + | > loss_disc: 2.7343759536743164 (2.7480641538873694) + | > loss_disc_real_0: 0.2367730736732483 (0.22089640223627607) + | > loss_disc_real_1: 0.1939273625612259 (0.23045159920389435) + | > loss_disc_real_2: 0.2417808473110199 (0.23447729096624065) + | > loss_disc_real_3: 0.20801319181919098 (0.23567096940402327) + | > loss_disc_real_4: 0.2557408809661865 (0.23593564617809992) + | > loss_disc_real_5: 0.2349463403224945 (0.2385424613365399) + | > loss_0: 2.7343759536743164 (2.7480641538873694) + | > grad_norm_0: tensor(5.0458, device='cuda:0') (tensor(11.2887, device='cuda:0')) + | > loss_gen: 1.727614164352417 (1.8758994798942152) + | > loss_kl: 2.450598955154419 (2.526096210103903) + | > loss_feat: 3.0939602851867676 (3.154502256750472) + | > loss_mel: 22.684728622436523 (22.21583788265736) + | > loss_duration: 1.3848778009414673 (1.3812108239516843) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.341779708862305 (31.153546638676687) + | > grad_norm_1: tensor(191.1825, device='cuda:0') (tensor(139.4945, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.8679 (0.8343787698322915) + | > loader_time: 0.0128 (0.011082298062705052) + + + --> TIME: 2024-06-25 05:38:54 -- STEP: 228/591 -- GLOBAL_STEP: 10275 + | > loss_disc: 2.819748878479004 (2.747319587490016) + | > loss_disc_real_0: 0.20800071954727173 (0.22092126290264882) + | > loss_disc_real_1: 0.2206566482782364 (0.23008307304821524) + | > loss_disc_real_2: 0.2644704580307007 (0.23440160432405638) + | > loss_disc_real_3: 0.20232659578323364 (0.23519689177996234) + | > loss_disc_real_4: 0.22851896286010742 (0.23582559261928526) + | > loss_disc_real_5: 0.2630433738231659 (0.23868940450381815) + | > loss_0: 2.819748878479004 (2.747319587490016) + | > grad_norm_0: tensor(8.8307, device='cuda:0') (tensor(11.5883, device='cuda:0')) + | > loss_gen: 1.7047207355499268 (1.8763555717049984) + | > loss_kl: 2.3691763877868652 (2.522114347993281) + | > loss_feat: 3.054537534713745 (3.151557485262552) + | > loss_mel: 22.003183364868164 (22.218563096565116) + | > loss_duration: 1.3718833923339844 (1.3809360087963578) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.503501892089844 (31.149526520779258) + | > grad_norm_1: tensor(91.3389, device='cuda:0') (tensor(134.6992, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.8921 (0.8391496643685457) + | > loader_time: 0.0143 (0.01137948768180713) + + + --> TIME: 2024-06-25 05:39:17 -- STEP: 253/591 -- GLOBAL_STEP: 10300 + | > loss_disc: 2.7136142253875732 (2.747724138229733) + | > loss_disc_real_0: 0.22521889209747314 (0.22052622183038312) + | > loss_disc_real_1: 0.19426804780960083 (0.23010437538030126) + | > loss_disc_real_2: 0.20379696786403656 (0.23438546909645142) + | > loss_disc_real_3: 0.18641790747642517 (0.23526088852185034) + | > loss_disc_real_4: 0.27442407608032227 (0.23609279543750372) + | > loss_disc_real_5: 0.24989096820354462 (0.2388123864006148) + | > loss_0: 2.7136142253875732 (2.747724138229733) + | > grad_norm_0: tensor(6.5282, device='cuda:0') (tensor(11.4373, device='cuda:0')) + | > loss_gen: 1.872677206993103 (1.8746245011039402) + | > loss_kl: 2.449902296066284 (2.520287087783511) + | > loss_feat: 2.921854257583618 (3.1405769553580294) + | > loss_mel: 21.8072566986084 (22.19088750394437) + | > loss_duration: 1.3705536127090454 (1.3812652265601484) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.422243118286133 (31.10764128794312) + | > grad_norm_1: tensor(59.5280, device='cuda:0') (tensor(130.2630, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.8874 (0.8444209720777428) + | > loader_time: 0.0156 (0.011714128637502314) + + + --> TIME: 2024-06-25 05:39:40 -- STEP: 278/591 -- GLOBAL_STEP: 10325 + | > loss_disc: 2.72216796875 (2.747256615179049) + | > loss_disc_real_0: 0.21313601732254028 (0.2200186698020791) + | > loss_disc_real_1: 0.26486220955848694 (0.23056318723469335) + | > loss_disc_real_2: 0.20414422452449799 (0.2346714091279524) + | > loss_disc_real_3: 0.27522987127304077 (0.2354897891553186) + | > loss_disc_real_4: 0.2502412497997284 (0.23590234490178472) + | > loss_disc_real_5: 0.2719588875770569 (0.23866426628485) + | > loss_0: 2.72216796875 (2.747256615179049) + | > grad_norm_0: tensor(4.7916, device='cuda:0') (tensor(11.2538, device='cuda:0')) + | > loss_gen: 1.8433756828308105 (1.8750878418092247) + | > loss_kl: 2.5511534214019775 (2.5186801462722332) + | > loss_feat: 3.0473480224609375 (3.1396018832707564) + | > loss_mel: 22.030187606811523 (22.174481000831662) + | > loss_duration: 1.3893197774887085 (1.381469240720324) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.861385345458984 (31.089320107329662) + | > grad_norm_1: tensor(75.0037, device='cuda:0') (tensor(129.3155, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.8974 (0.8492171807254818) + | > loader_time: 0.0163 (0.01199279157377833) + + + --> TIME: 2024-06-25 05:40:03 -- STEP: 303/591 -- GLOBAL_STEP: 10350 + | > loss_disc: 2.766209602355957 (2.747107156432502) + | > loss_disc_real_0: 0.30237799882888794 (0.22009678622676987) + | > loss_disc_real_1: 0.2383066862821579 (0.23053851644984966) + | > loss_disc_real_2: 0.23057414591312408 (0.23491718071718815) + | > loss_disc_real_3: 0.20990917086601257 (0.23539954084571046) + | > loss_disc_real_4: 0.21587301790714264 (0.23562310827840674) + | > loss_disc_real_5: 0.22822430729866028 (0.2383269388781916) + | > loss_0: 2.766209602355957 (2.747107156432502) + | > grad_norm_0: tensor(16.9340, device='cuda:0') (tensor(11.2789, device='cuda:0')) + | > loss_gen: 1.7188811302185059 (1.8738691157633716) + | > loss_kl: 2.5731875896453857 (2.519111028992302) + | > loss_feat: 3.130481004714966 (3.1368248462676993) + | > loss_mel: 22.15324592590332 (22.149968433694866) + | > loss_duration: 1.4189481735229492 (1.382097989419112) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.99474334716797 (31.06187141846509) + | > grad_norm_1: tensor(129.7349, device='cuda:0') (tensor(127.8926, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.8975 (0.8552812648685061) + | > loader_time: 0.0139 (0.012246688993850558) + + + --> TIME: 2024-06-25 05:40:27 -- STEP: 328/591 -- GLOBAL_STEP: 10375 + | > loss_disc: 2.7275452613830566 (2.7460133549643726) + | > loss_disc_real_0: 0.27545052766799927 (0.21975553258344893) + | > loss_disc_real_1: 0.21944858133792877 (0.2302539691601585) + | > loss_disc_real_2: 0.2511865794658661 (0.2350478488043314) + | > loss_disc_real_3: 0.27307355403900146 (0.23526410008894233) + | > loss_disc_real_4: 0.24775484204292297 (0.23550823916930977) + | > loss_disc_real_5: 0.25450873374938965 (0.23824846785424686) + | > loss_0: 2.7275452613830566 (2.7460133549643726) + | > grad_norm_0: tensor(13.0522, device='cuda:0') (tensor(11.1999, device='cuda:0')) + | > loss_gen: 1.9262322187423706 (1.874174763880125) + | > loss_kl: 2.4126532077789307 (2.5160583606580404) + | > loss_feat: 2.9904353618621826 (3.1364175735450366) + | > loss_mel: 21.600353240966797 (22.151672700556308) + | > loss_duration: 1.3873646259307861 (1.382789379212914) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.31703758239746 (31.061112741144694) + | > grad_norm_1: tensor(85.7440, device='cuda:0') (tensor(128.2087, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.945 (0.8612715732760542) + | > loader_time: 0.016 (0.012493491172790527) + + + --> TIME: 2024-06-25 05:40:51 -- STEP: 353/591 -- GLOBAL_STEP: 10400 + | > loss_disc: 2.7389674186706543 (2.746640845668891) + | > loss_disc_real_0: 0.1880790889263153 (0.21977588380278001) + | > loss_disc_real_1: 0.23885223269462585 (0.23042050983345208) + | > loss_disc_real_2: 0.2422141581773758 (0.23516055164863975) + | > loss_disc_real_3: 0.27655649185180664 (0.23533507075593424) + | > loss_disc_real_4: 0.227558434009552 (0.23551092290506823) + | > loss_disc_real_5: 0.2414831966161728 (0.23827587642176631) + | > loss_0: 2.7389674186706543 (2.746640845668891) + | > grad_norm_0: tensor(6.3071, device='cuda:0') (tensor(11.1988, device='cuda:0')) + | > loss_gen: 1.8446259498596191 (1.8732307777864061) + | > loss_kl: 2.429107427597046 (2.513589815782757) + | > loss_feat: 3.191903591156006 (3.1295469111153484) + | > loss_mel: 22.240028381347656 (22.13917439922731) + | > loss_duration: 1.3943324089050293 (1.3841313940945146) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.09999656677246 (31.039673273016305) + | > grad_norm_1: tensor(123.6011, device='cuda:0') (tensor(128.1497, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.9518 (0.8671924378986734) + | > loader_time: 0.0163 (0.012722532742421957) + + + --> TIME: 2024-06-25 05:41:16 -- STEP: 378/591 -- GLOBAL_STEP: 10425 + | > loss_disc: 2.707458734512329 (2.747429559470486) + | > loss_disc_real_0: 0.19472281634807587 (0.22008483565200573) + | > loss_disc_real_1: 0.2068769484758377 (0.23036772596142283) + | > loss_disc_real_2: 0.2456539422273636 (0.23518967013510447) + | > loss_disc_real_3: 0.19838181138038635 (0.23526211149951137) + | > loss_disc_real_4: 0.21240194141864777 (0.23550409058887492) + | > loss_disc_real_5: 0.22533001005649567 (0.23827463695927273) + | > loss_0: 2.707458734512329 (2.747429559470486) + | > grad_norm_0: tensor(6.3883, device='cuda:0') (tensor(11.4005, device='cuda:0')) + | > loss_gen: 1.8446316719055176 (1.8723949070960757) + | > loss_kl: 2.4393484592437744 (2.5114689119278437) + | > loss_feat: 2.884204387664795 (3.121385277894438) + | > loss_mel: 22.230815887451172 (22.122920026224108) + | > loss_duration: 1.4043086767196655 (1.3856866249331716) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.803308486938477 (31.01385573230723) + | > grad_norm_1: tensor(93.8844, device='cuda:0') (tensor(128.2972, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.9596 (0.8727195546740575) + | > loader_time: 0.0178 (0.01297498758507784) + + + --> TIME: 2024-06-25 05:41:40 -- STEP: 403/591 -- GLOBAL_STEP: 10450 + | > loss_disc: 2.7114052772521973 (2.748283315828954) + | > loss_disc_real_0: 0.16372105479240417 (0.21968073499557694) + | > loss_disc_real_1: 0.2696536183357239 (0.23050561835688932) + | > loss_disc_real_2: 0.24150918424129486 (0.235581410508002) + | > loss_disc_real_3: 0.24072876572608948 (0.23546653634709105) + | > loss_disc_real_4: 0.22391580045223236 (0.23556980157904234) + | > loss_disc_real_5: 0.2472618818283081 (0.23830233044718985) + | > loss_0: 2.7114052772521973 (2.748283315828954) + | > grad_norm_0: tensor(8.2118, device='cuda:0') (tensor(11.1369, device='cuda:0')) + | > loss_gen: 1.9949764013290405 (1.87103839725182) + | > loss_kl: 2.401730537414551 (2.508499563184033) + | > loss_feat: 3.1116061210632324 (3.113284558279638) + | > loss_mel: 22.56930923461914 (22.12502437430633) + | > loss_duration: 1.4321496486663818 (1.3866840772534121) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.5097713470459 (31.00453094217381) + | > grad_norm_1: tensor(262.4425, device='cuda:0') (tensor(129.3589, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.9524 (0.8790315019877315) + | > loader_time: 0.0166 (0.013249181340409272) + + + --> TIME: 2024-06-25 05:42:06 -- STEP: 428/591 -- GLOBAL_STEP: 10475 + | > loss_disc: 2.7735743522644043 (2.748250734026188) + | > loss_disc_real_0: 0.2206968069076538 (0.21946764542399166) + | > loss_disc_real_1: 0.2284233570098877 (0.23069743376886737) + | > loss_disc_real_2: 0.1931012123823166 (0.23548495908763922) + | > loss_disc_real_3: 0.23668453097343445 (0.23538917682577518) + | > loss_disc_real_4: 0.2185509353876114 (0.23558289738737534) + | > loss_disc_real_5: 0.20959173142910004 (0.23837116155251165) + | > loss_0: 2.7735743522644043 (2.748250734026188) + | > grad_norm_0: tensor(5.1935, device='cuda:0') (tensor(10.8815, device='cuda:0')) + | > loss_gen: 2.060018301010132 (1.8706260161979176) + | > loss_kl: 2.486403703689575 (2.5052222486968354) + | > loss_feat: 3.2333383560180664 (3.1098282476451904) + | > loss_mel: 22.858951568603516 (22.10736791218551) + | > loss_duration: 1.401964783668518 (1.3873041222028637) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.04067611694336 (30.98034849790769) + | > grad_norm_1: tensor(137.1474, device='cuda:0') (tensor(129.0482, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 0.9791 (0.8852870063246965) + | > loader_time: 0.0185 (0.013507361166945127) + + + --> TIME: 2024-06-25 05:42:31 -- STEP: 453/591 -- GLOBAL_STEP: 10500 + | > loss_disc: 2.802443742752075 (2.7494251985949933) + | > loss_disc_real_0: 0.22558429837226868 (0.21962483881851433) + | > loss_disc_real_1: 0.23960541188716888 (0.23087863308309722) + | > loss_disc_real_2: 0.23387077450752258 (0.23544240922196286) + | > loss_disc_real_3: 0.22186534106731415 (0.2355983140847541) + | > loss_disc_real_4: 0.2479703724384308 (0.2356984339013005) + | > loss_disc_real_5: 0.2634871304035187 (0.23843451357440443) + | > loss_0: 2.802443742752075 (2.7494251985949933) + | > grad_norm_0: tensor(4.1675, device='cuda:0') (tensor(10.9528, device='cuda:0')) + | > loss_gen: 1.8573966026306152 (1.868954506930926) + | > loss_kl: 2.439208984375 (2.5032376690416145) + | > loss_feat: 2.731806516647339 (3.1000716833615667) + | > loss_mel: 21.497379302978516 (22.088942327773054) + | > loss_duration: 1.405530333518982 (1.3882755467433798) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.931324005126953 (30.949481703061405) + | > grad_norm_1: tensor(47.9922, device='cuda:0') (tensor(127.1839, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 1.0022 (0.8919690555294615) + | > loader_time: 0.0163 (0.013780216507564317) + + + --> TIME: 2024-06-25 05:42:58 -- STEP: 478/591 -- GLOBAL_STEP: 10525 + | > loss_disc: 2.794032096862793 (2.750384884399351) + | > loss_disc_real_0: 0.11988092958927155 (0.21985901999735433) + | > loss_disc_real_1: 0.2758822441101074 (0.23097951251966686) + | > loss_disc_real_2: 0.23287616670131683 (0.23571042523224484) + | > loss_disc_real_3: 0.19427651166915894 (0.2356561795188792) + | > loss_disc_real_4: 0.22375349700450897 (0.23559502194242) + | > loss_disc_real_5: 0.26988354325294495 (0.2384485077097326) + | > loss_0: 2.794032096862793 (2.750384884399351) + | > grad_norm_0: tensor(20.1393, device='cuda:0') (tensor(11.0706, device='cuda:0')) + | > loss_gen: 1.6989530324935913 (1.8681934954731036) + | > loss_kl: 2.5018062591552734 (2.500368643505304) + | > loss_feat: 2.8287038803100586 (3.0931474928077787) + | > loss_mel: 21.632661819458008 (22.064823382070358) + | > loss_duration: 1.3850128650665283 (1.3895113355943842) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.04714012145996 (30.916044330995952) + | > grad_norm_1: tensor(60.7575, device='cuda:0') (tensor(126.4610, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 1.0373 (0.8991385299291569) + | > loader_time: 0.0199 (0.01407681548944577) + + + --> TIME: 2024-06-25 05:43:25 -- STEP: 503/591 -- GLOBAL_STEP: 10550 + | > loss_disc: 2.7206459045410156 (2.751095661349136) + | > loss_disc_real_0: 0.2710423469543457 (0.21996808189817263) + | > loss_disc_real_1: 0.1939821094274521 (0.2308505226082167) + | > loss_disc_real_2: 0.23563119769096375 (0.2357955118474619) + | > loss_disc_real_3: 0.2145942598581314 (0.23571652511716126) + | > loss_disc_real_4: 0.20011478662490845 (0.23553238112931224) + | > loss_disc_real_5: 0.21137937903404236 (0.23838484429459922) + | > loss_0: 2.7206459045410156 (2.751095661349136) + | > grad_norm_0: tensor(9.9417, device='cuda:0') (tensor(11.0295, device='cuda:0')) + | > loss_gen: 1.8238941431045532 (1.8670374510539456) + | > loss_kl: 2.500192403793335 (2.4981893924311187) + | > loss_feat: 2.967531204223633 (3.084347482704026) + | > loss_mel: 22.280895233154297 (22.04554587377467) + | > loss_duration: 1.4421395063400269 (1.390842374466048) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.014652252197266 (30.885962548834193) + | > grad_norm_1: tensor(240.8938, device='cuda:0') (tensor(127.1196, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 1.0546 (0.9079719542509043) + | > loader_time: 0.0203 (0.014381660855788124) + + + --> TIME: 2024-06-25 05:43:54 -- STEP: 528/591 -- GLOBAL_STEP: 10575 + | > loss_disc: 2.816361427307129 (2.7518320914470795) + | > loss_disc_real_0: 0.1658630073070526 (0.21983792580607717) + | > loss_disc_real_1: 0.26399415731430054 (0.23106634777716614) + | > loss_disc_real_2: 0.28885626792907715 (0.23592596334602797) + | > loss_disc_real_3: 0.2346142828464508 (0.23568313842582883) + | > loss_disc_real_4: 0.25590381026268005 (0.2357069608106306) + | > loss_disc_real_5: 0.23086512088775635 (0.23853612369434402) + | > loss_0: 2.816361427307129 (2.7518320914470795) + | > grad_norm_0: tensor(14.0297, device='cuda:0') (tensor(10.9537, device='cuda:0')) + | > loss_gen: 1.8528038263320923 (1.8658432682806796) + | > loss_kl: 2.567962169647217 (2.49578247738607) + | > loss_feat: 2.779118776321411 (3.0755421568950014) + | > loss_mel: 21.509700775146484 (22.022202079946332) + | > loss_duration: 1.45518159866333 (1.392299505797299) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.164766311645508 (30.85166946685675) + | > grad_norm_1: tensor(118.4669, device='cuda:0') (tensor(127.7547, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 1.1263 (0.917977911504832) + | > loader_time: 0.0233 (0.01470342143015428) + + + --> TIME: 2024-06-25 05:44:25 -- STEP: 553/591 -- GLOBAL_STEP: 10600 + | > loss_disc: 2.750885009765625 (2.752102085306676) + | > loss_disc_real_0: 0.21939504146575928 (0.21983708919549727) + | > loss_disc_real_1: 0.20339632034301758 (0.23114288586173307) + | > loss_disc_real_2: 0.24469417333602905 (0.23587752081063082) + | > loss_disc_real_3: 0.1977703720331192 (0.23571408603333652) + | > loss_disc_real_4: 0.2414504885673523 (0.23573819217802602) + | > loss_disc_real_5: 0.24209538102149963 (0.23865948172848653) + | > loss_0: 2.750885009765625 (2.752102085306676) + | > grad_norm_0: tensor(4.4984, device='cuda:0') (tensor(10.8186, device='cuda:0')) + | > loss_gen: 1.7478419542312622 (1.864527028464927) + | > loss_kl: 2.5041933059692383 (2.493861303622641) + | > loss_feat: 2.7780275344848633 (3.0665996635154182) + | > loss_mel: 21.398513793945312 (22.00173587695598) + | > loss_duration: 1.4329136610031128 (1.3937711250070632) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.86149024963379 (30.820494953590103) + | > grad_norm_1: tensor(175.8947, device='cuda:0') (tensor(125.7999, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 1.3105 (0.9315591207680172) + | > loader_time: 0.0242 (0.015091281687372633) + + + --> TIME: 2024-06-25 05:44:57 -- STEP: 578/591 -- GLOBAL_STEP: 10625 + | > loss_disc: 2.730293035507202 (2.7529794629469886) + | > loss_disc_real_0: 0.28997477889060974 (0.2200172351331654) + | > loss_disc_real_1: 0.21602751314640045 (0.2311740698080162) + | > loss_disc_real_2: 0.2249547690153122 (0.23595443455611959) + | > loss_disc_real_3: 0.25812771916389465 (0.2358670343989732) + | > loss_disc_real_4: 0.2178329974412918 (0.23563499256298204) + | > loss_disc_real_5: 0.24681395292282104 (0.23852797164533376) + | > loss_0: 2.730293035507202 (2.7529794629469886) + | > grad_norm_0: tensor(17.5196, device='cuda:0') (tensor(10.9120, device='cuda:0')) + | > loss_gen: 1.7904155254364014 (1.8637046898525074) + | > loss_kl: 2.5052788257598877 (2.4914186033823102) + | > loss_feat: 2.9221534729003906 (3.0567353303869687) + | > loss_mel: 21.13566017150879 (21.985017179205325) + | > loss_duration: 1.444351077079773 (1.3953682231655575) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.7978572845459 (30.792243964119354) + | > grad_norm_1: tensor(106.2795, device='cuda:0') (tensor(126.8285, device='cuda:0')) + | > current_lr_0: 0.00019957542473449108 + | > current_lr_1: 0.00019957542473449108 + | > step_time: 1.1646 (0.945293807240918) + | > loader_time: 0.0225 (0.015509711417359877) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003675805761459026 (+7.521345260295444e-05) + | > avg_loss_disc: 2.7578214787422337 (-0.011354517429433386) + | > avg_loss_disc_real_0: 0.18909648552219915 (-0.1013830499129092) + | > avg_loss_disc_real_1: 0.2725398134677969 (+0.07124838993904448) + | > avg_loss_disc_real_2: 0.2174272787697772 (-0.04510765284933943) + | > avg_loss_disc_real_3: 0.18957541439127415 (-0.006801342076443617) + | > avg_loss_disc_real_4: 0.21556295517911303 (+0.006089951446715841) + | > avg_loss_disc_real_5: 0.20871244053891364 (-0.012033417503884508) + | > avg_loss_0: 2.7578214787422337 (-0.011354517429433386) + | > avg_loss_gen: 1.7052602590398585 (-0.10879160495514584) + | > avg_loss_kl: 2.5102433803233692 (-0.041067985778159954) + | > avg_loss_feat: 3.017486019337431 (+0.0026187896728515625) + | > avg_loss_mel: 21.945032119750977 (-0.2904220743382169) + | > avg_loss_duration: 1.383616028948033 (-0.004754730995665257) + | > avg_loss_1: 30.561637837836084 (-0.4424175505942465) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_10638.pth + + > EPOCH: 19/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 05:45:27)  + + --> TIME: 2024-06-25 05:45:39 -- STEP: 12/591 -- GLOBAL_STEP: 10650 + | > loss_disc: 2.731208086013794 (2.737843374411265) + | > loss_disc_real_0: 0.17429128289222717 (0.21439425026377043) + | > loss_disc_real_1: 0.28400787711143494 (0.2279637654622396) + | > loss_disc_real_2: 0.21919609606266022 (0.23670155306657156) + | > loss_disc_real_3: 0.18687781691551208 (0.24157009522120157) + | > loss_disc_real_4: 0.20791442692279816 (0.23659261564413706) + | > loss_disc_real_5: 0.22706672549247742 (0.23804687584439913) + | > loss_0: 2.731208086013794 (2.737843374411265) + | > grad_norm_0: tensor(11.1798, device='cuda:0') (tensor(7.4226, device='cuda:0')) + | > loss_gen: 1.909180998802185 (1.8889567752679188) + | > loss_kl: 2.4093098640441895 (2.622960607210795) + | > loss_feat: 3.4429237842559814 (3.238499859968821) + | > loss_mel: 22.473892211914062 (22.28947861989339) + | > loss_duration: 1.3365689516067505 (1.3546757698059082) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.571876525878906 (31.394572099049885) + | > grad_norm_1: tensor(150.6910, device='cuda:0') (tensor(155.0034, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.772 (0.7702125708262125) + | > loader_time: 0.009 (0.008573333422342936) + + + --> TIME: 2024-06-25 05:45:59 -- STEP: 37/591 -- GLOBAL_STEP: 10675 + | > loss_disc: 2.748770236968994 (2.7404184534743026) + | > loss_disc_real_0: 0.3133155107498169 (0.22268242570194038) + | > loss_disc_real_1: 0.24241062998771667 (0.2282802458550479) + | > loss_disc_real_2: 0.2280406504869461 (0.2341291719191783) + | > loss_disc_real_3: 0.2130947709083557 (0.23746048558402705) + | > loss_disc_real_4: 0.2415713667869568 (0.2356768246438052) + | > loss_disc_real_5: 0.21018870174884796 (0.2372084066674516) + | > loss_0: 2.748770236968994 (2.7404184534743026) + | > grad_norm_0: tensor(17.2627, device='cuda:0') (tensor(11.1067, device='cuda:0')) + | > loss_gen: 1.7587001323699951 (1.88709062498969) + | > loss_kl: 2.4190480709075928 (2.5937258037360937) + | > loss_feat: 3.050565719604492 (3.244124779830108) + | > loss_mel: 20.807397842407227 (22.059195750468486) + | > loss_duration: 1.3563590049743652 (1.360244641432891) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.392070770263672 (31.14438175510716) + | > grad_norm_1: tensor(77.1874, device='cuda:0') (tensor(136.5995, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.8113 (0.7859021264153558) + | > loader_time: 0.0115 (0.009466280808319917) + + + --> TIME: 2024-06-25 05:46:20 -- STEP: 62/591 -- GLOBAL_STEP: 10700 + | > loss_disc: 2.7537684440612793 (2.743912750674832) + | > loss_disc_real_0: 0.24369753897190094 (0.2220076548476373) + | > loss_disc_real_1: 0.2609415650367737 (0.22958287885112147) + | > loss_disc_real_2: 0.22968371212482452 (0.23375494273439532) + | > loss_disc_real_3: 0.23298940062522888 (0.23605427458401648) + | > loss_disc_real_4: 0.23189781606197357 (0.23627773335864466) + | > loss_disc_real_5: 0.24284051358699799 (0.23833911072823308) + | > loss_0: 2.7537684440612793 (2.743912750674832) + | > grad_norm_0: tensor(6.3857, device='cuda:0') (tensor(9.8501, device='cuda:0')) + | > loss_gen: 1.7977665662765503 (1.8781839936010298) + | > loss_kl: 2.594167709350586 (2.5715454932182076) + | > loss_feat: 3.0777175426483154 (3.2008265910610074) + | > loss_mel: 23.068593978881836 (22.020720881800496) + | > loss_duration: 1.4138317108154297 (1.3701263377743382) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.952075958251953 (31.04140333975515) + | > grad_norm_1: tensor(124.7995, device='cuda:0') (tensor(131.4344, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.8095 (0.7999321606851393) + | > loader_time: 0.0101 (0.009893494267617502) + + + --> TIME: 2024-06-25 05:46:41 -- STEP: 87/591 -- GLOBAL_STEP: 10725 + | > loss_disc: 2.706902265548706 (2.7452784023065675) + | > loss_disc_real_0: 0.22544527053833008 (0.22236727469268885) + | > loss_disc_real_1: 0.2137548178434372 (0.22897975198153792) + | > loss_disc_real_2: 0.22322823107242584 (0.23376388899211226) + | > loss_disc_real_3: 0.27633482217788696 (0.23545973431104902) + | > loss_disc_real_4: 0.2314857691526413 (0.23576821386814117) + | > loss_disc_real_5: 0.24608492851257324 (0.23844337223590106) + | > loss_0: 2.706902265548706 (2.7452784023065675) + | > grad_norm_0: tensor(5.9921, device='cuda:0') (tensor(10.0259, device='cuda:0')) + | > loss_gen: 1.8234975337982178 (1.8740279592316726) + | > loss_kl: 2.481079339981079 (2.5605009670915275) + | > loss_feat: 3.016115188598633 (3.1896370580826683) + | > loss_mel: 21.981409072875977 (22.041763174122774) + | > loss_duration: 1.4320294857025146 (1.3776967004797924) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.734130859375 (31.043625842565778) + | > grad_norm_1: tensor(144.8126, device='cuda:0') (tensor(135.7120, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.837 (0.8083485954109279) + | > loader_time: 0.0116 (0.010335165878822064) + + + --> TIME: 2024-06-25 05:47:02 -- STEP: 112/591 -- GLOBAL_STEP: 10750 + | > loss_disc: 2.752626895904541 (2.7446286784751073) + | > loss_disc_real_0: 0.20003342628479004 (0.220882689048137) + | > loss_disc_real_1: 0.25452205538749695 (0.22920097716684854) + | > loss_disc_real_2: 0.21894784271717072 (0.23386728005217655) + | > loss_disc_real_3: 0.2919847071170807 (0.23561381069677217) + | > loss_disc_real_4: 0.2363487333059311 (0.23571011942944356) + | > loss_disc_real_5: 0.24375371634960175 (0.2386277008003422) + | > loss_0: 2.752626895904541 (2.7446286784751073) + | > grad_norm_0: tensor(5.6875, device='cuda:0') (tensor(10.0815, device='cuda:0')) + | > loss_gen: 2.0074257850646973 (1.875296066914286) + | > loss_kl: 2.4481418132781982 (2.545518102390426) + | > loss_feat: 3.1392877101898193 (3.190345266035625) + | > loss_mel: 22.23680305480957 (22.072702748434885) + | > loss_duration: 1.4156427383422852 (1.3816288049731935) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.247299194335938 (31.06549092701503) + | > grad_norm_1: tensor(81.6290, device='cuda:0') (tensor(136.5670, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.8295 (0.8142933909382137) + | > loader_time: 0.0113 (0.010641926101275853) + + + --> TIME: 2024-06-25 05:47:24 -- STEP: 137/591 -- GLOBAL_STEP: 10775 + | > loss_disc: 2.6814732551574707 (2.7446098066594478) + | > loss_disc_real_0: 0.21321386098861694 (0.2207981887525015) + | > loss_disc_real_1: 0.24782001972198486 (0.2294134586831949) + | > loss_disc_real_2: 0.21387168765068054 (0.23371387615691136) + | > loss_disc_real_3: 0.2239922732114792 (0.23516716432832455) + | > loss_disc_real_4: 0.2533915042877197 (0.2357725155179518) + | > loss_disc_real_5: 0.2605155408382416 (0.23853081540904775) + | > loss_0: 2.6814732551574707 (2.7446098066594478) + | > grad_norm_0: tensor(6.1368, device='cuda:0') (tensor(10.0715, device='cuda:0')) + | > loss_gen: 1.9000475406646729 (1.8713618560429037) + | > loss_kl: 2.461240768432617 (2.5381452480371856) + | > loss_feat: 3.1945698261260986 (3.1813233124948765) + | > loss_mel: 21.976011276245117 (22.066376456379025) + | > loss_duration: 1.369178295135498 (1.3823115608117875) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.901046752929688 (31.03951838416775) + | > grad_norm_1: tensor(271.3051, device='cuda:0') (tensor(136.5120, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.8549 (0.8198091409502237) + | > loader_time: 0.0121 (0.010852758031691948) + + + --> TIME: 2024-06-25 05:47:45 -- STEP: 162/591 -- GLOBAL_STEP: 10800 + | > loss_disc: 2.7646238803863525 (2.7439596756005002) + | > loss_disc_real_0: 0.2775639295578003 (0.22058309462887266) + | > loss_disc_real_1: 0.22951006889343262 (0.2290906513362755) + | > loss_disc_real_2: 0.2567281126976013 (0.23376532322094765) + | > loss_disc_real_3: 0.2426963597536087 (0.2352873937767229) + | > loss_disc_real_4: 0.1937728226184845 (0.23536016994420392) + | > loss_disc_real_5: 0.20492464303970337 (0.23844037738479215) + | > loss_0: 2.7646238803863525 (2.7439596756005002) + | > grad_norm_0: tensor(9.6913, device='cuda:0') (tensor(10.2773, device='cuda:0')) + | > loss_gen: 1.9446513652801514 (1.8735773342627067) + | > loss_kl: 2.4699690341949463 (2.5348754988776316) + | > loss_feat: 3.2506725788116455 (3.1890173267435142) + | > loss_mel: 22.244722366333008 (22.046704563093773) + | > loss_duration: 1.3698313236236572 (1.3804857385011366) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.27984619140625 (31.024660416591313) + | > grad_norm_1: tensor(220.5115, device='cuda:0') (tensor(133.9472, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.8712 (0.8255690674722931) + | > loader_time: 0.0129 (0.011031584975160198) + + + --> TIME: 2024-06-25 05:48:07 -- STEP: 187/591 -- GLOBAL_STEP: 10825 + | > loss_disc: 2.805345296859741 (2.7461875759981544) + | > loss_disc_real_0: 0.13622966408729553 (0.22113743304887554) + | > loss_disc_real_1: 0.2675911784172058 (0.22947124164690946) + | > loss_disc_real_2: 0.24416691064834595 (0.2340813418601286) + | > loss_disc_real_3: 0.15889579057693481 (0.2351534306046797) + | > loss_disc_real_4: 0.18934166431427002 (0.23531042286418977) + | > loss_disc_real_5: 0.19687658548355103 (0.23829976975280334) + | > loss_0: 2.805345296859741 (2.7461875759981544) + | > grad_norm_0: tensor(18.5489, device='cuda:0') (tensor(10.5117, device='cuda:0')) + | > loss_gen: 1.8358906507492065 (1.8737718612752496) + | > loss_kl: 2.5743069648742676 (2.5340878937971136) + | > loss_feat: 3.1529881954193115 (3.1854761794289166) + | > loss_mel: 21.86112403869629 (22.045900497844503) + | > loss_duration: 1.3562363386154175 (1.3792881704269246) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.78054428100586 (31.01852454731171) + | > grad_norm_1: tensor(86.7490, device='cuda:0') (tensor(133.3375, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.8646 (0.8306604821414234) + | > loader_time: 0.0126 (0.011129021007109453) + + + --> TIME: 2024-06-25 05:48:30 -- STEP: 212/591 -- GLOBAL_STEP: 10850 + | > loss_disc: 2.7719788551330566 (2.7451777435698608) + | > loss_disc_real_0: 0.2748294472694397 (0.22069563744765405) + | > loss_disc_real_1: 0.22347226738929749 (0.22920648205392766) + | > loss_disc_real_2: 0.2106691300868988 (0.23401763016041718) + | > loss_disc_real_3: 0.2561706006526947 (0.2354790265408327) + | > loss_disc_real_4: 0.2583909034729004 (0.2352641302178491) + | > loss_disc_real_5: 0.2289445400238037 (0.23837529579704664) + | > loss_0: 2.7719788551330566 (2.7451777435698608) + | > grad_norm_0: tensor(11.1092, device='cuda:0') (tensor(10.2187, device='cuda:0')) + | > loss_gen: 1.9735808372497559 (1.8737447216825664) + | > loss_kl: 2.4222404956817627 (2.527472858159046) + | > loss_feat: 3.0319690704345703 (3.1809853776445927) + | > loss_mel: 21.568483352661133 (22.037155394284223) + | > loss_duration: 1.3469940423965454 (1.378269485707553) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.34326934814453 (30.997627780122578) + | > grad_norm_1: tensor(99.8402, device='cuda:0') (tensor(131.7240, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.8641 (0.8356595792860355) + | > loader_time: 0.0123 (0.011328211370504127) + + + --> TIME: 2024-06-25 05:48:52 -- STEP: 237/591 -- GLOBAL_STEP: 10875 + | > loss_disc: 2.704059362411499 (2.745127025032849) + | > loss_disc_real_0: 0.2693234980106354 (0.22080321943206624) + | > loss_disc_real_1: 0.2173394113779068 (0.22947604765620414) + | > loss_disc_real_2: 0.23937799036502838 (0.23419013411938389) + | > loss_disc_real_3: 0.2739810049533844 (0.2353042819957693) + | > loss_disc_real_4: 0.21801449358463287 (0.23529107384540865) + | > loss_disc_real_5: 0.23109015822410583 (0.2382491014677764) + | > loss_0: 2.704059362411499 (2.745127025032849) + | > grad_norm_0: tensor(9.6978, device='cuda:0') (tensor(10.4411, device='cuda:0')) + | > loss_gen: 1.8040661811828613 (1.8723529858931207) + | > loss_kl: 2.5369858741760254 (2.5263471301597877) + | > loss_feat: 3.133268356323242 (3.1733654853160873) + | > loss_mel: 21.68370819091797 (22.029474210135536) + | > loss_duration: 1.3775372505187988 (1.3781748300866237) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.535564422607422 (30.979714590788895) + | > grad_norm_1: tensor(78.1057, device='cuda:0') (tensor(128.3611, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.9136 (0.8405458293383634) + | > loader_time: 0.0146 (0.011571223222756688) + + + --> TIME: 2024-06-25 05:49:15 -- STEP: 262/591 -- GLOBAL_STEP: 10900 + | > loss_disc: 2.7769179344177246 (2.745031608880022) + | > loss_disc_real_0: 0.21617582440376282 (0.22016469992071616) + | > loss_disc_real_1: 0.20786349475383759 (0.22946736751166918) + | > loss_disc_real_2: 0.22526776790618896 (0.23408118282793133) + | > loss_disc_real_3: 0.2224954217672348 (0.23531963403215844) + | > loss_disc_real_4: 0.21105405688285828 (0.23517619687637298) + | > loss_disc_real_5: 0.21257132291793823 (0.23844282920578963) + | > loss_0: 2.7769179344177246 (2.745031608880022) + | > grad_norm_0: tensor(5.6016, device='cuda:0') (tensor(10.1361, device='cuda:0')) + | > loss_gen: 1.8988615274429321 (1.8715402279191344) + | > loss_kl: 2.318627119064331 (2.522355499158378) + | > loss_feat: 3.1517562866210938 (3.171483407493766) + | > loss_mel: 22.136247634887695 (22.019684405727233) + | > loss_duration: 1.3581962585449219 (1.377864011826406) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.863689422607422 (30.962927505260204) + | > grad_norm_1: tensor(116.2990, device='cuda:0') (tensor(125.9094, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.8901 (0.84542480133872) + | > loader_time: 0.0127 (0.011823220107391591) + + + --> TIME: 2024-06-25 05:49:38 -- STEP: 287/591 -- GLOBAL_STEP: 10925 + | > loss_disc: 2.784825325012207 (2.745268353186419) + | > loss_disc_real_0: 0.23594026267528534 (0.22031898786174292) + | > loss_disc_real_1: 0.22629079222679138 (0.2292255208363517) + | > loss_disc_real_2: 0.22824256122112274 (0.23410270209927178) + | > loss_disc_real_3: 0.16447602212429047 (0.23543016830595528) + | > loss_disc_real_4: 0.20313899219036102 (0.23520577078496954) + | > loss_disc_real_5: 0.20765341818332672 (0.23846415593856718) + | > loss_0: 2.784825325012207 (2.745268353186419) + | > grad_norm_0: tensor(8.8881, device='cuda:0') (tensor(10.1166, device='cuda:0')) + | > loss_gen: 1.8464570045471191 (1.8708961404574458) + | > loss_kl: 2.4757368564605713 (2.5173744507367597) + | > loss_feat: 3.1446852684020996 (3.168080451181126) + | > loss_mel: 22.12798500061035 (21.999451786798872) + | > loss_duration: 1.3847949504852295 (1.3784687037251966) + | > amp_scaler: 1024.0 (520.9198606271777) + | > loss_1: 30.979658126831055 (30.934271476825355) + | > grad_norm_1: tensor(63.8085, device='cuda:0') (tensor(123.4155, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.9145 (0.8501371838905255) + | > loader_time: 0.0136 (0.012107105620646727) + + + --> TIME: 2024-06-25 05:50:02 -- STEP: 312/591 -- GLOBAL_STEP: 10950 + | > loss_disc: 2.7605369091033936 (2.745361794263889) + | > loss_disc_real_0: 0.2182159423828125 (0.22021389258309051) + | > loss_disc_real_1: 0.23426243662834167 (0.22943269265576818) + | > loss_disc_real_2: 0.22068892419338226 (0.23412404515040228) + | > loss_disc_real_3: 0.2117394357919693 (0.23540571288993725) + | > loss_disc_real_4: 0.24259665608406067 (0.2352272609774119) + | > loss_disc_real_5: 0.25096362829208374 (0.23853585133567834) + | > loss_0: 2.7605369091033936 (2.745361794263889) + | > grad_norm_0: tensor(4.6336, device='cuda:0') (tensor(10.0144, device='cuda:0')) + | > loss_gen: 1.8739451169967651 (1.870948694073237) + | > loss_kl: 2.5405776500701904 (2.518385378214028) + | > loss_feat: 3.178978443145752 (3.1645334049677243) + | > loss_mel: 22.568248748779297 (21.993625426903744) + | > loss_duration: 1.387707233428955 (1.3789001454909635) + | > amp_scaler: 1024.0 (561.2307692307688) + | > loss_1: 31.549455642700195 (30.926392970941006) + | > grad_norm_1: tensor(66.5938, device='cuda:0') (tensor(123.8675, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.9542 (0.8557230883683914) + | > loader_time: 0.0165 (0.012418957092823127) + + + --> TIME: 2024-06-25 05:50:25 -- STEP: 337/591 -- GLOBAL_STEP: 10975 + | > loss_disc: 2.775031089782715 (2.746839847338306) + | > loss_disc_real_0: 0.19888845086097717 (0.2208089672183778) + | > loss_disc_real_1: 0.21877798438072205 (0.22957390723667093) + | > loss_disc_real_2: 0.20587888360023499 (0.23419910200625568) + | > loss_disc_real_3: 0.21277567744255066 (0.23527485176434504) + | > loss_disc_real_4: 0.21214520931243896 (0.23514262867610602) + | > loss_disc_real_5: 0.22364084422588348 (0.23848152346356333) + | > loss_0: 2.775031089782715 (2.746839847338306) + | > grad_norm_0: tensor(6.8432, device='cuda:0') (tensor(10.1213, device='cuda:0')) + | > loss_gen: 1.832837462425232 (1.8697155756483446) + | > loss_kl: 2.573629856109619 (2.5180870156613566) + | > loss_feat: 3.0261189937591553 (3.1580220223888107) + | > loss_mel: 21.729150772094727 (21.985512934381944) + | > loss_duration: 1.397594928741455 (1.3799094792997086) + | > amp_scaler: 512.0 (594.0415430267058) + | > loss_1: 30.5593318939209 (30.911246970428557) + | > grad_norm_1: 0 (tensor(121.7857, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.8967 (0.8610559464915923) + | > loader_time: 0.0174 (0.012679238701432678) + + + --> TIME: 2024-06-25 05:50:49 -- STEP: 362/591 -- GLOBAL_STEP: 11000 + | > loss_disc: 2.7115085124969482 (2.7467110216288293) + | > loss_disc_real_0: 0.26310673356056213 (0.22078603018301626) + | > loss_disc_real_1: 0.269844651222229 (0.22963170682527748) + | > loss_disc_real_2: 0.23217007517814636 (0.23436036693621737) + | > loss_disc_real_3: 0.26801955699920654 (0.23535677053816412) + | > loss_disc_real_4: 0.2800453007221222 (0.23531744155765238) + | > loss_disc_real_5: 0.24805817008018494 (0.23860218330641478) + | > loss_0: 2.7115085124969482 (2.7467110216288293) + | > grad_norm_0: tensor(9.6161, device='cuda:0') (tensor(9.8254, device='cuda:0')) + | > loss_gen: 1.7029144763946533 (1.868435248156279) + | > loss_kl: 2.424967050552368 (2.5156760913891016) + | > loss_feat: 3.1283655166625977 (3.1501186987313132) + | > loss_mel: 22.258108139038086 (21.972991616686393) + | > loss_duration: 1.4182204008102417 (1.3812890971563139) + | > amp_scaler: 512.0 (588.3756906077343) + | > loss_1: 30.932575225830078 (30.888510709309447) + | > grad_norm_1: tensor(214.8175, device='cuda:0') (tensor(120.8674, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.936 (0.8666990297275353) + | > loader_time: 0.017 (0.012949425871200983) + + + --> TIME: 2024-06-25 05:51:14 -- STEP: 387/591 -- GLOBAL_STEP: 11025 + | > loss_disc: 2.7914605140686035 (2.7479915471040015) + | > loss_disc_real_0: 0.327028751373291 (0.22084640934870842) + | > loss_disc_real_1: 0.2215076982975006 (0.22969670874819895) + | > loss_disc_real_2: 0.24855969846248627 (0.23443430731462878) + | > loss_disc_real_3: 0.2697676718235016 (0.23549317855397553) + | > loss_disc_real_4: 0.24887339770793915 (0.2352728622289283) + | > loss_disc_real_5: 0.23952718079090118 (0.23864071452340416) + | > loss_0: 2.7914605140686035 (2.7479915471040015) + | > grad_norm_0: tensor(17.9780, device='cuda:0') (tensor(9.8834, device='cuda:0')) + | > loss_gen: 1.7427998781204224 (1.8675739377043967) + | > loss_kl: 2.4803318977355957 (2.5135130198426947) + | > loss_feat: 2.86346697807312 (3.143543430693083) + | > loss_mel: 21.633243560791016 (21.956871752283053) + | > loss_duration: 1.3593376874923706 (1.3823170603399746) + | > amp_scaler: 512.0 (583.4418604651158) + | > loss_1: 30.079179763793945 (30.86381915188575) + | > grad_norm_1: tensor(37.5085, device='cuda:0') (tensor(120.0797, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.9743 (0.8727024500992255) + | > loader_time: 0.0177 (0.013194531433342038) + + + --> TIME: 2024-06-25 05:51:39 -- STEP: 412/591 -- GLOBAL_STEP: 11050 + | > loss_disc: 2.7588863372802734 (2.747537497756552) + | > loss_disc_real_0: 0.31718823313713074 (0.22059093880494243) + | > loss_disc_real_1: 0.2332647442817688 (0.22950721774431113) + | > loss_disc_real_2: 0.2397557497024536 (0.2344205702465136) + | > loss_disc_real_3: 0.19228146970272064 (0.23548322326494653) + | > loss_disc_real_4: 0.260319322347641 (0.2352992928071508) + | > loss_disc_real_5: 0.23537585139274597 (0.23876678549404284) + | > loss_0: 2.7588863372802734 (2.747537497756552) + | > grad_norm_0: tensor(20.3612, device='cuda:0') (tensor(9.7502, device='cuda:0')) + | > loss_gen: 1.6547784805297852 (1.8666534750785642) + | > loss_kl: 2.66070818901062 (2.511813484927982) + | > loss_feat: 2.9348762035369873 (3.137825301549968) + | > loss_mel: 21.403162002563477 (21.943812629551555) + | > loss_duration: 1.3831005096435547 (1.3831078314665453) + | > amp_scaler: 512.0 (579.1067961165043) + | > loss_1: 30.0366268157959 (30.84321268785347) + | > grad_norm_1: tensor(43.8539, device='cuda:0') (tensor(120.2406, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 0.9827 (0.8795724239164185) + | > loader_time: 0.021 (0.013484487834485989) + + + --> TIME: 2024-06-25 05:52:04 -- STEP: 437/591 -- GLOBAL_STEP: 11075 + | > loss_disc: 2.7043418884277344 (2.7492678956527197) + | > loss_disc_real_0: 0.22443170845508575 (0.22135151561255859) + | > loss_disc_real_1: 0.163925439119339 (0.22949481914054343) + | > loss_disc_real_2: 0.24625630676746368 (0.23450427602985086) + | > loss_disc_real_3: 0.26595544815063477 (0.23545144900713688) + | > loss_disc_real_4: 0.25713714957237244 (0.23521491530558064) + | > loss_disc_real_5: 0.24213974177837372 (0.23868811021407627) + | > loss_0: 2.7043418884277344 (2.7492678956527197) + | > grad_norm_0: tensor(5.1673, device='cuda:0') (tensor(9.9085, device='cuda:0')) + | > loss_gen: 1.8779804706573486 (1.866159200668335) + | > loss_kl: 2.4425392150878906 (2.5108501196453163) + | > loss_feat: 3.0112862586975098 (3.128857887582322) + | > loss_mel: 22.814332962036133 (21.929423434783438) + | > loss_duration: 1.396921992301941 (1.3838696569818107) + | > amp_scaler: 512.0 (575.2677345537751) + | > loss_1: 31.543060302734375 (30.819160265016883) + | > grad_norm_1: tensor(250.1892, device='cuda:0') (tensor(121.3681, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 1.026 (0.8861190934475827) + | > loader_time: 0.0196 (0.013759219673732871) + + + --> TIME: 2024-06-25 05:52:30 -- STEP: 462/591 -- GLOBAL_STEP: 11100 + | > loss_disc: 2.762737274169922 (2.751113988620379) + | > loss_disc_real_0: 0.2759910225868225 (0.22127375490479656) + | > loss_disc_real_1: 0.310119092464447 (0.23029244978190502) + | > loss_disc_real_2: 0.22179366648197174 (0.23483388625827187) + | > loss_disc_real_3: 0.2890676259994507 (0.23556067830040342) + | > loss_disc_real_4: 0.23769895732402802 (0.23535438017411667) + | > loss_disc_real_5: 0.2515742778778076 (0.2387455818908555) + | > loss_0: 2.762737274169922 (2.751113988620379) + | > grad_norm_0: tensor(10.3925, device='cuda:0') (tensor(9.7901, device='cuda:0')) + | > loss_gen: 1.7656750679016113 (1.8657421010397213) + | > loss_kl: 2.3773748874664307 (2.5092139140868075) + | > loss_feat: 2.931124687194824 (3.1194701953367763) + | > loss_mel: 21.91204261779785 (21.917380939830426) + | > loss_duration: 1.4221251010894775 (1.385054985166112) + | > amp_scaler: 512.0 (571.8441558441551) + | > loss_1: 30.408342361450195 (30.796862098561736) + | > grad_norm_1: tensor(188.9148, device='cuda:0') (tensor(124.8519, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 1.0249 (0.8928269447186292) + | > loader_time: 0.0172 (0.014023343722025553) + + + --> TIME: 2024-06-25 05:52:57 -- STEP: 487/591 -- GLOBAL_STEP: 11125 + | > loss_disc: 2.8102097511291504 (2.7513261002926375) + | > loss_disc_real_0: 0.28005605936050415 (0.2213125607071471) + | > loss_disc_real_1: 0.21226154267787933 (0.23010955867214128) + | > loss_disc_real_2: 0.24635812640190125 (0.23480208028390911) + | > loss_disc_real_3: 0.18325576186180115 (0.2355469835489927) + | > loss_disc_real_4: 0.23761995136737823 (0.23534222069584612) + | > loss_disc_real_5: 0.24331094324588776 (0.23872641331728478) + | > loss_0: 2.8102097511291504 (2.7513261002926375) + | > grad_norm_0: tensor(10.1445, device='cuda:0') (tensor(9.8447, device='cuda:0')) + | > loss_gen: 1.8470275402069092 (1.8646044023717454) + | > loss_kl: 2.4723570346832275 (2.5067286755759604) + | > loss_feat: 3.007632255554199 (3.112478219018581) + | > loss_mel: 22.006122589111328 (21.904631393401278) + | > loss_duration: 1.408129334449768 (1.3864493355369176) + | > amp_scaler: 512.0 (568.7720739219706) + | > loss_1: 30.74126625061035 (30.774891980619646) + | > grad_norm_1: tensor(61.0983, device='cuda:0') (tensor(123.9738, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 1.0455 (0.9001747127431129) + | > loader_time: 0.024 (0.014314104644180078) + + + --> TIME: 2024-06-25 05:53:24 -- STEP: 512/591 -- GLOBAL_STEP: 11150 + | > loss_disc: 2.7808544635772705 (2.7510412959381947) + | > loss_disc_real_0: 0.20281767845153809 (0.2210233559308108) + | > loss_disc_real_1: 0.27623361349105835 (0.23014252682332884) + | > loss_disc_real_2: 0.2767188549041748 (0.23475095586036332) + | > loss_disc_real_3: 0.22770430147647858 (0.23551626343396492) + | > loss_disc_real_4: 0.21730947494506836 (0.2353629321733024) + | > loss_disc_real_5: 0.21258197724819183 (0.23862345848465338) + | > loss_0: 2.7808544635772705 (2.7510412959381947) + | > grad_norm_0: tensor(5.4872, device='cuda:0') (tensor(9.6642, device='cuda:0')) + | > loss_gen: 1.8427956104278564 (1.8632777640596032) + | > loss_kl: 2.367471218109131 (2.5048859156668177) + | > loss_feat: 2.9993231296539307 (3.105226602870972) + | > loss_mel: 22.141706466674805 (21.88933061808347) + | > loss_duration: 1.4703278541564941 (1.3881420984398574) + | > amp_scaler: 512.0 (565.9999999999993) + | > loss_1: 30.821624755859375 (30.750862941145897) + | > grad_norm_1: tensor(110.6628, device='cuda:0') (tensor(123.3450, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 1.0582 (0.9080328773707151) + | > loader_time: 0.0217 (0.01463733660057187) + + + --> TIME: 2024-06-25 05:53:52 -- STEP: 537/591 -- GLOBAL_STEP: 11175 + | > loss_disc: 2.792323589324951 (2.7510807935974015) + | > loss_disc_real_0: 0.1665595918893814 (0.22107656005929527) + | > loss_disc_real_1: 0.295790433883667 (0.2301132679540574) + | > loss_disc_real_2: 0.24215145409107208 (0.23467145874069611) + | > loss_disc_real_3: 0.2799263894557953 (0.2354880135152593) + | > loss_disc_real_4: 0.2058972716331482 (0.23533651560156482) + | > loss_disc_real_5: 0.2414553314447403 (0.2386719624423448) + | > loss_0: 2.792323589324951 (2.7510807935974015) + | > grad_norm_0: tensor(8.7943, device='cuda:0') (tensor(9.8022, device='cuda:0')) + | > loss_gen: 1.840722680091858 (1.8629053071890462) + | > loss_kl: 2.4579315185546875 (2.5023984056611295) + | > loss_feat: 2.8477065563201904 (3.099459236560588) + | > loss_mel: 21.967098236083984 (21.87665587696949) + | > loss_duration: 1.427425742149353 (1.389614699273136) + | > amp_scaler: 512.0 (563.4860335195524) + | > loss_1: 30.540884017944336 (30.73103346371784) + | > grad_norm_1: tensor(137.1794, device='cuda:0') (tensor(124.0659, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 1.118 (0.9164597299955854) + | > loader_time: 0.0265 (0.014979701246407223) + + + --> TIME: 2024-06-25 05:54:21 -- STEP: 562/591 -- GLOBAL_STEP: 11200 + | > loss_disc: 2.8267507553100586 (2.751952935792375) + | > loss_disc_real_0: 0.0951574519276619 (0.22118924060352552) + | > loss_disc_real_1: 0.24361824989318848 (0.23021714784390568) + | > loss_disc_real_2: 0.2526908218860626 (0.23485397796719953) + | > loss_disc_real_3: 0.23480834066867828 (0.2354720607930231) + | > loss_disc_real_4: 0.19911077618598938 (0.23534384706584585) + | > loss_disc_real_5: 0.21579937636852264 (0.2386411822987621) + | > loss_0: 2.8267507553100586 (2.751952935792375) + | > grad_norm_0: tensor(30.7064, device='cuda:0') (tensor(9.9141, device='cuda:0')) + | > loss_gen: 1.767759084701538 (1.8623590265728824) + | > loss_kl: 2.4282615184783936 (2.500663573631612) + | > loss_feat: 3.1648266315460205 (3.092688767511226) + | > loss_mel: 21.338666915893555 (21.85450562718076) + | > loss_duration: 1.407809853553772 (1.391175085115263) + | > amp_scaler: 512.0 (561.1957295373659) + | > loss_1: 30.107324600219727 (30.701392017649585) + | > grad_norm_1: tensor(203.3447, device='cuda:0') (tensor(123.7344, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 1.1343 (0.9258734136289549) + | > loader_time: 0.0225 (0.015308220191358248) + + + --> TIME: 2024-06-25 05:54:51 -- STEP: 587/591 -- GLOBAL_STEP: 11225 + | > loss_disc: 2.783997058868408 (2.7519304285260655) + | > loss_disc_real_0: 0.2823505997657776 (0.22108323458943913) + | > loss_disc_real_1: 0.21313297748565674 (0.23020362899697422) + | > loss_disc_real_2: 0.24243822693824768 (0.23494998569992173) + | > loss_disc_real_3: 0.2843160927295685 (0.2354730977415023) + | > loss_disc_real_4: 0.24538284540176392 (0.23552487770276337) + | > loss_disc_real_5: 0.2070426493883133 (0.2387646210000706) + | > loss_0: 2.783997058868408 (2.7519304285260655) + | > grad_norm_0: tensor(13.2539, device='cuda:0') (tensor(9.8004, device='cuda:0')) + | > loss_gen: 1.6562873125076294 (1.8616013447714421) + | > loss_kl: 2.540477991104126 (2.4975890569020773) + | > loss_feat: 2.895726442337036 (3.0836710881130864) + | > loss_mel: 21.673433303833008 (21.83908427431799) + | > loss_duration: 1.4324688911437988 (1.3929180377995949) + | > amp_scaler: 512.0 (559.1005110732534) + | > loss_1: 30.198394775390625 (30.674863750321293) + | > grad_norm_1: tensor(92.1581, device='cuda:0') (tensor(124.0845, device='cuda:0')) + | > current_lr_0: 0.00019955047780639926 + | > current_lr_1: 0.00019955047780639926 + | > step_time: 1.1511 (0.9368778485455749) + | > loader_time: 0.0225 (0.01567305170088448) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037978608557518493 (+0.00012205509429282337) + | > avg_loss_disc: 2.6993076649117977 (-0.05851381383043597) + | > avg_loss_disc_real_0: 0.23326805709524356 (+0.04417157157304441) + | > avg_loss_disc_real_1: 0.208793284728172 (-0.06374652873962489) + | > avg_loss_disc_real_2: 0.25240517930781586 (+0.034977900538038675) + | > avg_loss_disc_real_3: 0.2048790045875184 (+0.015303590196244249) + | > avg_loss_disc_real_4: 0.2425377866689195 (+0.026974831489806478) + | > avg_loss_disc_real_5: 0.26818808247434334 (+0.0594756419354297) + | > avg_loss_0: 2.6993076649117977 (-0.05851381383043597) + | > avg_loss_gen: 1.8936800905998716 (+0.18841983156001318) + | > avg_loss_kl: 2.529606920607547 (+0.01936354028417764) + | > avg_loss_feat: 3.065505565480983 (+0.04801954614355175) + | > avg_loss_mel: 21.889791894466313 (-0.055240225284663325) + | > avg_loss_duration: 1.392597239068214 (+0.008981210120180938) + | > avg_loss_1: 30.771181755877556 (+0.2095439180414722) + + + > EPOCH: 20/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 05:55:04)  + + --> TIME: 2024-06-25 05:55:22 -- STEP: 21/591 -- GLOBAL_STEP: 11250 + | > loss_disc: 2.7774171829223633 (2.735845474969773) + | > loss_disc_real_0: 0.22137776017189026 (0.2128280812785739) + | > loss_disc_real_1: 0.25279873609542847 (0.23489263086091905) + | > loss_disc_real_2: 0.2844536006450653 (0.2335886238586335) + | > loss_disc_real_3: 0.25255152583122253 (0.238263895823842) + | > loss_disc_real_4: 0.24148955941200256 (0.2368397450163251) + | > loss_disc_real_5: 0.23377060890197754 (0.23755737429573423) + | > loss_0: 2.7774171829223633 (2.735845474969773) + | > grad_norm_0: tensor(4.4803, device='cuda:0') (tensor(10.2967, device='cuda:0')) + | > loss_gen: 2.0259227752685547 (1.8942579825719197) + | > loss_kl: 2.732083797454834 (2.6699036075955345) + | > loss_feat: 3.3270461559295654 (3.3039289883204868) + | > loss_mel: 22.50798988342285 (22.039286295572918) + | > loss_duration: 1.3870105743408203 (1.3572886784871419) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.980052947998047 (31.264665603637695) + | > grad_norm_1: tensor(172.1174, device='cuda:0') (tensor(96.8967, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.7954 (0.7759666897001722) + | > loader_time: 0.0099 (0.0087018126533145) + + + --> TIME: 2024-06-25 05:55:43 -- STEP: 46/591 -- GLOBAL_STEP: 11275 + | > loss_disc: 2.727208137512207 (2.745862271474756) + | > loss_disc_real_0: 0.25153863430023193 (0.21873970264973847) + | > loss_disc_real_1: 0.26373645663261414 (0.23145764191513477) + | > loss_disc_real_2: 0.2234094738960266 (0.23384078106154566) + | > loss_disc_real_3: 0.21672360599040985 (0.23795773672020953) + | > loss_disc_real_4: 0.26069316267967224 (0.23654004866662232) + | > loss_disc_real_5: 0.2355635166168213 (0.23847054625334946) + | > loss_0: 2.727208137512207 (2.745862271474756) + | > grad_norm_0: tensor(6.1291, device='cuda:0') (tensor(10.9914, device='cuda:0')) + | > loss_gen: 1.951783537864685 (1.8790485858917236) + | > loss_kl: 2.6275646686553955 (2.6256128964216816) + | > loss_feat: 3.3226547241210938 (3.2260378029035484) + | > loss_mel: 21.957809448242188 (21.947116063988727) + | > loss_duration: 1.3751163482666016 (1.3631329147712046) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.234928131103516 (31.040948370228644) + | > grad_norm_1: tensor(189.2310, device='cuda:0') (tensor(130.8769, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.821 (0.7892138232355534) + | > loader_time: 0.0093 (0.009147669958031698) + + + --> TIME: 2024-06-25 05:56:03 -- STEP: 71/591 -- GLOBAL_STEP: 11300 + | > loss_disc: 2.8628129959106445 (2.753242435589643) + | > loss_disc_real_0: 0.15763485431671143 (0.21908535262648487) + | > loss_disc_real_1: 0.28497570753097534 (0.23463605218370195) + | > loss_disc_real_2: 0.22685721516609192 (0.23454134648954364) + | > loss_disc_real_3: 0.1991756707429886 (0.23624434626438248) + | > loss_disc_real_4: 0.22591647505760193 (0.2356688294192435) + | > loss_disc_real_5: 0.20606721937656403 (0.23717251986684934) + | > loss_0: 2.8628129959106445 (2.753242435589643) + | > grad_norm_0: tensor(15.4099, device='cuda:0') (tensor(11.3324, device='cuda:0')) + | > loss_gen: 1.8589038848876953 (1.8824529127335885) + | > loss_kl: 2.5050854682922363 (2.590428819119091) + | > loss_feat: 3.185725688934326 (3.221066001435401) + | > loss_mel: 22.100994110107422 (21.947185220852703) + | > loss_duration: 1.378994107246399 (1.3700883992960755) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.02970314025879 (31.01122136183188) + | > grad_norm_1: tensor(137.3177, device='cuda:0') (tensor(154.5013, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.8201 (0.7978376670622491) + | > loader_time: 0.0103 (0.00957398683252469) + + + --> TIME: 2024-06-25 05:56:25 -- STEP: 96/591 -- GLOBAL_STEP: 11325 + | > loss_disc: 2.747973918914795 (2.7604547515511517) + | > loss_disc_real_0: 0.23117421567440033 (0.21951664402149618) + | > loss_disc_real_1: 0.24160990118980408 (0.23689386760815978) + | > loss_disc_real_2: 0.23462769389152527 (0.23634407157078385) + | > loss_disc_real_3: 0.24072390794754028 (0.23596854787319899) + | > loss_disc_real_4: 0.2315388023853302 (0.23593853258838257) + | > loss_disc_real_5: 0.24157975614070892 (0.2385017485357821) + | > loss_0: 2.747973918914795 (2.7604547515511517) + | > grad_norm_0: tensor(3.5931, device='cuda:0') (tensor(10.6782, device='cuda:0')) + | > loss_gen: 1.7428936958312988 (1.877048134803772) + | > loss_kl: 2.596266746520996 (2.576185849805673) + | > loss_feat: 3.122807741165161 (3.1931894744435945) + | > loss_mel: 21.69632339477539 (22.032455702622737) + | > loss_duration: 1.396240234375 (1.3777481578290462) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.55453109741211 (31.056627253691357) + | > grad_norm_1: tensor(52.9405, device='cuda:0') (tensor(149.8369, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.8224 (0.8057713930805525) + | > loader_time: 0.0109 (0.009936039646466574) + + + --> TIME: 2024-06-25 05:56:46 -- STEP: 121/591 -- GLOBAL_STEP: 11350 + | > loss_disc: 2.8020663261413574 (2.7575528109369203) + | > loss_disc_real_0: 0.3008798360824585 (0.21933301956939302) + | > loss_disc_real_1: 0.2407190203666687 (0.23518599816097702) + | > loss_disc_real_2: 0.26839783787727356 (0.2363916729107376) + | > loss_disc_real_3: 0.206074059009552 (0.23600200668346782) + | > loss_disc_real_4: 0.23746885359287262 (0.23559266279551608) + | > loss_disc_real_5: 0.22612667083740234 (0.23858371216896151) + | > loss_0: 2.8020663261413574 (2.7575528109369203) + | > grad_norm_0: tensor(13.6755, device='cuda:0') (tensor(10.2390, device='cuda:0')) + | > loss_gen: 1.7780027389526367 (1.873170348238354) + | > loss_kl: 2.5318596363067627 (2.562353197208121) + | > loss_feat: 3.2156476974487305 (3.1906048955996176) + | > loss_mel: 21.981679916381836 (21.985230422217004) + | > loss_duration: 1.3773601055145264 (1.3793863462022513) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.884550094604492 (30.9907451503533) + | > grad_norm_1: tensor(210.5362, device='cuda:0') (tensor(143.5292, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.831 (0.8121879691920004) + | > loader_time: 0.0107 (0.01015826296215215) + + + --> TIME: 2024-06-25 05:57:07 -- STEP: 146/591 -- GLOBAL_STEP: 11375 + | > loss_disc: 2.7434093952178955 (2.7523216384730933) + | > loss_disc_real_0: 0.16757848858833313 (0.21846231925365042) + | > loss_disc_real_1: 0.19787698984146118 (0.23335841573672753) + | > loss_disc_real_2: 0.24352388083934784 (0.2357065125483356) + | > loss_disc_real_3: 0.23391222953796387 (0.2358503067126013) + | > loss_disc_real_4: 0.2795734703540802 (0.23546838872644998) + | > loss_disc_real_5: 0.2726605534553528 (0.23915005097650502) + | > loss_0: 2.7434093952178955 (2.7523216384730933) + | > grad_norm_0: tensor(10.2579, device='cuda:0') (tensor(9.9443, device='cuda:0')) + | > loss_gen: 1.8800690174102783 (1.874078148031888) + | > loss_kl: 2.4981954097747803 (2.5534146501593393) + | > loss_feat: 3.0855724811553955 (3.195696866675599) + | > loss_mel: 21.466623306274414 (21.95084469938932) + | > loss_duration: 1.3588082790374756 (1.3789051828319079) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.289270401000977 (30.95293947768538) + | > grad_norm_1: tensor(89.9295, device='cuda:0') (tensor(136.7222, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.878 (0.8182871586655918) + | > loader_time: 0.0113 (0.010361124391425147) + + + --> TIME: 2024-06-25 05:57:30 -- STEP: 171/591 -- GLOBAL_STEP: 11400 + | > loss_disc: 2.76163387298584 (2.749842309115226) + | > loss_disc_real_0: 0.2216925024986267 (0.21899036243993636) + | > loss_disc_real_1: 0.18560121953487396 (0.23242728159441586) + | > loss_disc_real_2: 0.22292867302894592 (0.23526060319783396) + | > loss_disc_real_3: 0.21619199216365814 (0.2355039294003046) + | > loss_disc_real_4: 0.27921146154403687 (0.23559757727279998) + | > loss_disc_real_5: 0.2694270610809326 (0.23872407774130502) + | > loss_0: 2.76163387298584 (2.749842309115226) + | > grad_norm_0: tensor(4.7430, device='cuda:0') (tensor(10.1857, device='cuda:0')) + | > loss_gen: 1.8748596906661987 (1.874272766866182) + | > loss_kl: 2.616405963897705 (2.5455283881628024) + | > loss_feat: 3.00484299659729 (3.2005874343782836) + | > loss_mel: 21.484046936035156 (21.928760015476517) + | > loss_duration: 1.3589478731155396 (1.37661397875401) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.33910369873047 (30.925762488828067) + | > grad_norm_1: tensor(149.8503, device='cuda:0') (tensor(134.3353, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 1.0948 (0.8257289880897567) + | > loader_time: 0.013 (0.010577030349196055) + + + --> TIME: 2024-06-25 05:57:52 -- STEP: 196/591 -- GLOBAL_STEP: 11425 + | > loss_disc: 2.7754626274108887 (2.749003947997579) + | > loss_disc_real_0: 0.2097873091697693 (0.21911069750785828) + | > loss_disc_real_1: 0.2552669048309326 (0.23207684241387308) + | > loss_disc_real_2: 0.2040424793958664 (0.23503998408512194) + | > loss_disc_real_3: 0.20807811617851257 (0.2356141510666633) + | > loss_disc_real_4: 0.24364221096038818 (0.2353552858136138) + | > loss_disc_real_5: 0.2492687702178955 (0.23854677371528685) + | > loss_0: 2.7754626274108887 (2.749003947997579) + | > grad_norm_0: tensor(6.2948, device='cuda:0') (tensor(10.0109, device='cuda:0')) + | > loss_gen: 1.724134087562561 (1.8741436503371414) + | > loss_kl: 2.48907470703125 (2.540757574597184) + | > loss_feat: 2.921844005584717 (3.201223755369381) + | > loss_mel: 21.682994842529297 (21.92283598257571) + | > loss_duration: 1.3817821741104126 (1.3757966343237424) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.199831008911133 (30.914757504755137) + | > grad_norm_1: tensor(188.3831, device='cuda:0') (tensor(133.6947, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.8683 (0.8317120610451212) + | > loader_time: 0.0135 (0.010845668461858015) + + + --> TIME: 2024-06-25 05:58:14 -- STEP: 221/591 -- GLOBAL_STEP: 11450 + | > loss_disc: 2.7339000701904297 (2.749621230552638) + | > loss_disc_real_0: 0.19267895817756653 (0.21956719882887413) + | > loss_disc_real_1: 0.2760782241821289 (0.23207808990554032) + | > loss_disc_real_2: 0.21541506052017212 (0.23494673253993642) + | > loss_disc_real_3: 0.24742984771728516 (0.23544337313908797) + | > loss_disc_real_4: 0.24835067987442017 (0.23520048488588893) + | > loss_disc_real_5: 0.21697889268398285 (0.23828835158326508) + | > loss_0: 2.7339000701904297 (2.749621230552638) + | > grad_norm_0: tensor(5.3839, device='cuda:0') (tensor(10.3483, device='cuda:0')) + | > loss_gen: 1.9587652683258057 (1.8747650050469653) + | > loss_kl: 2.4958584308624268 (2.5355686677527114) + | > loss_feat: 3.4101369380950928 (3.196754800787878) + | > loss_mel: 22.869415283203125 (21.923924821534307) + | > loss_duration: 1.4063060283660889 (1.3752251210795263) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.140480041503906 (30.906238288361564) + | > grad_norm_1: tensor(167.1598, device='cuda:0') (tensor(137.8891, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.8902 (0.8373020482818465) + | > loader_time: 0.013 (0.011172103666072521) + + + --> TIME: 2024-06-25 05:58:37 -- STEP: 246/591 -- GLOBAL_STEP: 11475 + | > loss_disc: 2.70622181892395 (2.7489874866919783) + | > loss_disc_real_0: 0.27423587441444397 (0.21940528995137873) + | > loss_disc_real_1: 0.1957615166902542 (0.231654001808748) + | > loss_disc_real_2: 0.22559866309165955 (0.23487151916918717) + | > loss_disc_real_3: 0.22008413076400757 (0.23547711938135024) + | > loss_disc_real_4: 0.19076775014400482 (0.23536144760323735) + | > loss_disc_real_5: 0.2000681310892105 (0.23842599830491756) + | > loss_0: 2.70622181892395 (2.7489874866919783) + | > grad_norm_0: tensor(11.2839, device='cuda:0') (tensor(10.0129, device='cuda:0')) + | > loss_gen: 1.795750379562378 (1.8736074615300187) + | > loss_kl: 2.551398754119873 (2.5304991937265173) + | > loss_feat: 3.3264520168304443 (3.1932225673179317) + | > loss_mel: 21.91997718811035 (21.936462092205762) + | > loss_duration: 1.394180178642273 (1.3753792396405844) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.98775863647461 (30.909170414374127) + | > grad_norm_1: tensor(139.7970, device='cuda:0') (tensor(135.9701, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.8912 (0.8423489952475075) + | > loader_time: 0.0146 (0.011488882506765974) + + + --> TIME: 2024-06-25 05:59:00 -- STEP: 271/591 -- GLOBAL_STEP: 11500 + | > loss_disc: 2.7467286586761475 (2.750254438372116) + | > loss_disc_real_0: 0.19330772757530212 (0.21933025666048606) + | > loss_disc_real_1: 0.20647144317626953 (0.23189173759789483) + | > loss_disc_real_2: 0.2259826511144638 (0.2352289226992104) + | > loss_disc_real_3: 0.2242668718099594 (0.23562429009548413) + | > loss_disc_real_4: 0.2381729781627655 (0.23541313811201892) + | > loss_disc_real_5: 0.24223105609416962 (0.2385261290297737) + | > loss_0: 2.7467286586761475 (2.750254438372116) + | > grad_norm_0: tensor(7.5203, device='cuda:0') (tensor(9.9355, device='cuda:0')) + | > loss_gen: 1.7800414562225342 (1.8725309332358442) + | > loss_kl: 2.486074447631836 (2.5276328347265955) + | > loss_feat: 3.0164549350738525 (3.1867226498593264) + | > loss_mel: 21.51475715637207 (21.925875991033013) + | > loss_duration: 1.3788970708847046 (1.3755752890752246) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.176223754882812 (30.88833755408706) + | > grad_norm_1: tensor(129.7825, device='cuda:0') (tensor(137.8191, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.9138 (0.8479921052376722) + | > loader_time: 0.0161 (0.011768451036122455) + + + --> TIME: 2024-06-25 05:59:23 -- STEP: 296/591 -- GLOBAL_STEP: 11525 + | > loss_disc: 2.73545241355896 (2.7495626831376874) + | > loss_disc_real_0: 0.28615930676460266 (0.2194460648539904) + | > loss_disc_real_1: 0.21443359553813934 (0.23166053102829973) + | > loss_disc_real_2: 0.23980873823165894 (0.2352544271160622) + | > loss_disc_real_3: 0.27038413286209106 (0.23551537514337012) + | > loss_disc_real_4: 0.25879696011543274 (0.2353322297536038) + | > loss_disc_real_5: 0.2570497691631317 (0.2384519398816534) + | > loss_0: 2.73545241355896 (2.7495626831376874) + | > grad_norm_0: tensor(15.0729, device='cuda:0') (tensor(9.9831, device='cuda:0')) + | > loss_gen: 1.716691255569458 (1.8715690833491248) + | > loss_kl: 2.4004228115081787 (2.526136052769586) + | > loss_feat: 3.143566846847534 (3.1822082134517466) + | > loss_mel: 22.601301193237305 (21.909704994510957) + | > loss_duration: 1.39580237865448 (1.3761945427269544) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.257783889770508 (30.865812759141665) + | > grad_norm_1: tensor(158.4181, device='cuda:0') (tensor(136.3494, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.8976 (0.8533339693739607) + | > loader_time: 0.0144 (0.01203427201992757) + + + --> TIME: 2024-06-25 05:59:47 -- STEP: 321/591 -- GLOBAL_STEP: 11550 + | > loss_disc: 2.670938014984131 (2.747557009491965) + | > loss_disc_real_0: 0.20416322350502014 (0.21914486814510786) + | > loss_disc_real_1: 0.2532086670398712 (0.23124384383360544) + | > loss_disc_real_2: 0.24790708720684052 (0.2349674587494859) + | > loss_disc_real_3: 0.30041518807411194 (0.23552401006407456) + | > loss_disc_real_4: 0.21998926997184753 (0.23523303425386316) + | > loss_disc_real_5: 0.24334333837032318 (0.2381992423645804) + | > loss_0: 2.670938014984131 (2.747557009491965) + | > grad_norm_0: tensor(5.3092, device='cuda:0') (tensor(9.8048, device='cuda:0')) + | > loss_gen: 1.9958127737045288 (1.8723119374747588) + | > loss_kl: 2.422361373901367 (2.523694336228656) + | > loss_feat: 3.4332759380340576 (3.1823220297554955) + | > loss_mel: 21.65542221069336 (21.902125777485203) + | > loss_duration: 1.4044163227081299 (1.376586551235472) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.91128921508789 (30.857040518169462) + | > grad_norm_1: tensor(80.9904, device='cuda:0') (tensor(136.6367, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.9892 (0.8587543511316412) + | > loader_time: 0.0157 (0.01227569802899227) + + + --> TIME: 2024-06-25 06:00:11 -- STEP: 346/591 -- GLOBAL_STEP: 11575 + | > loss_disc: 2.7417399883270264 (2.7468765539930047) + | > loss_disc_real_0: 0.20364704728126526 (0.21903028858104193) + | > loss_disc_real_1: 0.24848909676074982 (0.23127721732406947) + | > loss_disc_real_2: 0.2847398817539215 (0.23508051067935248) + | > loss_disc_real_3: 0.23427735269069672 (0.2353813492752224) + | > loss_disc_real_4: 0.2465479075908661 (0.23532824050312098) + | > loss_disc_real_5: 0.24036362767219543 (0.23821686604463985) + | > loss_0: 2.7417399883270264 (2.7468765539930047) + | > grad_norm_0: tensor(5.9702, device='cuda:0') (tensor(9.8943, device='cuda:0')) + | > loss_gen: 1.8010849952697754 (1.8719542526096278) + | > loss_kl: 2.502640962600708 (2.521917071645663) + | > loss_feat: 2.9831902980804443 (3.1793870030110982) + | > loss_mel: 21.71963119506836 (21.88922014401827) + | > loss_duration: 1.413466215133667 (1.3779323631628395) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.420013427734375 (30.840410750725365) + | > grad_norm_1: tensor(137.4356, device='cuda:0') (tensor(135.2727, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.9595 (0.8641822503481297) + | > loader_time: 0.0172 (0.01251620088698547) + + + --> TIME: 2024-06-25 06:00:35 -- STEP: 371/591 -- GLOBAL_STEP: 11600 + | > loss_disc: 2.888305425643921 (2.7482907123000158) + | > loss_disc_real_0: 0.3814387619495392 (0.21941762192631667) + | > loss_disc_real_1: 0.256028413772583 (0.2312119570623511) + | > loss_disc_real_2: 0.2329748421907425 (0.2350757963895155) + | > loss_disc_real_3: 0.2414846420288086 (0.23551278370570622) + | > loss_disc_real_4: 0.23088127374649048 (0.23523732015707421) + | > loss_disc_real_5: 0.21940864622592926 (0.23823708520745332) + | > loss_0: 2.888305425643921 (2.7482907123000158) + | > grad_norm_0: tensor(27.7075, device='cuda:0') (tensor(9.8617, device='cuda:0')) + | > loss_gen: 1.6696667671203613 (1.8714956691001625) + | > loss_kl: 2.4488131999969482 (2.5188401522983463) + | > loss_feat: 3.015047550201416 (3.172547981745472) + | > loss_mel: 22.227697372436523 (21.879226370963128) + | > loss_duration: 1.4213082790374756 (1.3793549682252164) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.782533645629883 (30.821465068107667) + | > grad_norm_1: tensor(70.2367, device='cuda:0') (tensor(135.5756, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.9633 (0.8700499894484034) + | > loader_time: 0.0167 (0.01278641088953558) + + + --> TIME: 2024-06-25 06:01:00 -- STEP: 396/591 -- GLOBAL_STEP: 11625 + | > loss_disc: 2.7323834896087646 (2.7480523911389447) + | > loss_disc_real_0: 0.1691068857908249 (0.2191402642150419) + | > loss_disc_real_1: 0.22384954988956451 (0.23135450964022164) + | > loss_disc_real_2: 0.2332213670015335 (0.23513851250813464) + | > loss_disc_real_3: 0.25091123580932617 (0.23550213937356015) + | > loss_disc_real_4: 0.2279318869113922 (0.23528135940432549) + | > loss_disc_real_5: 0.22400911152362823 (0.23819255956796684) + | > loss_0: 2.7323834896087646 (2.7480523911389447) + | > grad_norm_0: tensor(7.7948, device='cuda:0') (tensor(9.9344, device='cuda:0')) + | > loss_gen: 1.81471848487854 (1.8715044446665832) + | > loss_kl: 2.4959146976470947 (2.5181106353046934) + | > loss_feat: 3.1335465908050537 (3.1682953190321888) + | > loss_mel: 21.771135330200195 (21.87438984109898) + | > loss_duration: 1.4124728441238403 (1.3802572199792567) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.627788543701172 (30.812557374588167) + | > grad_norm_1: tensor(45.6717, device='cuda:0') (tensor(132.8830, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.9746 (0.8765351236468614) + | > loader_time: 0.0183 (0.013107424432581121) + + + --> TIME: 2024-06-25 06:01:25 -- STEP: 421/591 -- GLOBAL_STEP: 11650 + | > loss_disc: 2.7977185249328613 (2.7472152262572167) + | > loss_disc_real_0: 0.2793516516685486 (0.2192466213166572) + | > loss_disc_real_1: 0.20320017635822296 (0.23116720466036128) + | > loss_disc_real_2: 0.23289403319358826 (0.2349819345952779) + | > loss_disc_real_3: 0.23629209399223328 (0.23535035125858145) + | > loss_disc_real_4: 0.2376263290643692 (0.23511746879711287) + | > loss_disc_real_5: 0.24130366742610931 (0.23815513212862038) + | > loss_0: 2.7977185249328613 (2.7472152262572167) + | > grad_norm_0: tensor(11.7238, device='cuda:0') (tensor(9.8422, device='cuda:0')) + | > loss_gen: 1.7205995321273804 (1.8707001322522017) + | > loss_kl: 2.5344014167785645 (2.515303621382727) + | > loss_feat: 2.9349417686462402 (3.1619923947259547) + | > loss_mel: 20.828554153442383 (21.852770748727398) + | > loss_duration: 1.3948156833648682 (1.3809851056889508) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.413312911987305 (30.781751913582628) + | > grad_norm_1: tensor(110.0801, device='cuda:0') (tensor(133.9602, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 0.9817 (0.882912859497614) + | > loader_time: 0.0184 (0.013401457929271417) + + + --> TIME: 2024-06-25 06:01:51 -- STEP: 446/591 -- GLOBAL_STEP: 11675 + | > loss_disc: 2.703421115875244 (2.74673027307998) + | > loss_disc_real_0: 0.16474203765392303 (0.21918912544311964) + | > loss_disc_real_1: 0.16162636876106262 (0.23077272426654405) + | > loss_disc_real_2: 0.19984345138072968 (0.23491755041039045) + | > loss_disc_real_3: 0.16486051678657532 (0.23521320221135433) + | > loss_disc_real_4: 0.22817517817020416 (0.23512487200343554) + | > loss_disc_real_5: 0.2518427073955536 (0.23825515967046199) + | > loss_0: 2.703421115875244 (2.74673027307998) + | > grad_norm_0: tensor(14.7192, device='cuda:0') (tensor(9.9572, device='cuda:0')) + | > loss_gen: 1.9887803792953491 (1.8703946733153989) + | > loss_kl: 2.5436995029449463 (2.513810697692394) + | > loss_feat: 3.125760793685913 (3.159755332052976) + | > loss_mel: 21.461742401123047 (21.848593382557414) + | > loss_duration: 1.3899962902069092 (1.3821818914114088) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.509979248046875 (30.774735899784105) + | > grad_norm_1: tensor(142.4016, device='cuda:0') (tensor(134.3988, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 1.0333 (0.8899248875844639) + | > loader_time: 0.0151 (0.013675715356664273) + + + --> TIME: 2024-06-25 06:02:17 -- STEP: 471/591 -- GLOBAL_STEP: 11700 + | > loss_disc: 2.740878105163574 (2.7469343693646158) + | > loss_disc_real_0: 0.3078916072845459 (0.21933220867898054) + | > loss_disc_real_1: 0.2417801171541214 (0.23094715013498968) + | > loss_disc_real_2: 0.21474739909172058 (0.23488828549339513) + | > loss_disc_real_3: 0.2273496836423874 (0.23528086772137133) + | > loss_disc_real_4: 0.22131788730621338 (0.2352329854815882) + | > loss_disc_real_5: 0.23047196865081787 (0.23821021069133863) + | > loss_0: 2.740878105163574 (2.7469343693646158) + | > grad_norm_0: tensor(12.2744, device='cuda:0') (tensor(9.9910, device='cuda:0')) + | > loss_gen: 1.7892780303955078 (1.8699765828005068) + | > loss_kl: 2.4703004360198975 (2.5129239270641563) + | > loss_feat: 2.9941089153289795 (3.153575360648446) + | > loss_mel: 21.690916061401367 (21.830856558117393) + | > loss_duration: 1.413980484008789 (1.3831961769966559) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.358583450317383 (30.750528525901196) + | > grad_norm_1: tensor(50.2226, device='cuda:0') (tensor(134.1776, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 1.0506 (0.8967325333070856) + | > loader_time: 0.022 (0.013967884067770275) + + + --> TIME: 2024-06-25 06:02:44 -- STEP: 496/591 -- GLOBAL_STEP: 11725 + | > loss_disc: 2.7526376247406006 (2.746780809856231) + | > loss_disc_real_0: 0.1896892786026001 (0.21927550331419032) + | > loss_disc_real_1: 0.20423321425914764 (0.23083049353332288) + | > loss_disc_real_2: 0.2239716351032257 (0.23478148583202593) + | > loss_disc_real_3: 0.29920560121536255 (0.23544106250929256) + | > loss_disc_real_4: 0.21427591145038605 (0.2350981872108194) + | > loss_disc_real_5: 0.24377667903900146 (0.23814725449248667) + | > loss_0: 2.7526376247406006 (2.746780809856231) + | > grad_norm_0: tensor(5.1406, device='cuda:0') (tensor(10.0036, device='cuda:0')) + | > loss_gen: 1.844170331954956 (1.8694525361061096) + | > loss_kl: 2.4737937450408936 (2.5097806564261855) + | > loss_feat: 3.093093156814575 (3.1473774857098067) + | > loss_mel: 21.056800842285156 (21.803958465976102) + | > loss_duration: 1.4150915145874023 (1.3844816677512657) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.882949829101562 (30.71505073578127) + | > grad_norm_1: tensor(226.9999, device='cuda:0') (tensor(133.2204, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 1.1488 (0.9051617677173307) + | > loader_time: 0.017 (0.014285006830769201) + + + --> TIME: 2024-06-25 06:03:13 -- STEP: 521/591 -- GLOBAL_STEP: 11750 + | > loss_disc: 2.8370378017425537 (2.747867862459795) + | > loss_disc_real_0: 0.3231106996536255 (0.2194409599087975) + | > loss_disc_real_1: 0.23305845260620117 (0.23080331061371456) + | > loss_disc_real_2: 0.2362956702709198 (0.23488186199697103) + | > loss_disc_real_3: 0.23084662854671478 (0.23548713498060625) + | > loss_disc_real_4: 0.25118979811668396 (0.23519533815402216) + | > loss_disc_real_5: 0.24394653737545013 (0.23815898425633986) + | > loss_0: 2.8370378017425537 (2.747867862459795) + | > grad_norm_0: tensor(16.4879, device='cuda:0') (tensor(10.0800, device='cuda:0')) + | > loss_gen: 1.7805519104003906 (1.8685569264380808) + | > loss_kl: 2.4917633533477783 (2.507599231148866) + | > loss_feat: 2.692286968231201 (3.140118466228991) + | > loss_mel: 20.937341690063477 (21.79290686916711) + | > loss_duration: 1.40557861328125 (1.3860422851027998) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.30752182006836 (30.69522371127372) + | > grad_norm_1: tensor(161.0976, device='cuda:0') (tensor(133.8357, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 1.1192 (0.9153640247352293) + | > loader_time: 0.0233 (0.014591386359392338) + + + --> TIME: 2024-06-25 06:03:43 -- STEP: 546/591 -- GLOBAL_STEP: 11775 + | > loss_disc: 2.730937957763672 (2.747638472707281) + | > loss_disc_real_0: 0.2308036983013153 (0.21912497070519044) + | > loss_disc_real_1: 0.23721550405025482 (0.23076201219370954) + | > loss_disc_real_2: 0.220723494887352 (0.2349369391635224) + | > loss_disc_real_3: 0.18054458498954773 (0.23553726764825675) + | > loss_disc_real_4: 0.23254689574241638 (0.23522486152884725) + | > loss_disc_real_5: 0.2494756430387497 (0.23822929860253036) + | > loss_0: 2.730937957763672 (2.747638472707281) + | > grad_norm_0: tensor(5.5583, device='cuda:0') (tensor(10.0301, device='cuda:0')) + | > loss_gen: 1.854966163635254 (1.867781512684874) + | > loss_kl: 2.4117279052734375 (2.5048678567558) + | > loss_feat: 3.080909252166748 (3.1342561423996873) + | > loss_mel: 21.495471954345703 (21.77031428997334) + | > loss_duration: 1.4224801063537598 (1.3875021323179584) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.265554428100586 (30.664721887190264) + | > grad_norm_1: tensor(122.7971, device='cuda:0') (tensor(133.0919, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 1.2165 (0.9282072612217493) + | > loader_time: 0.0192 (0.014930757410797007) + + + --> TIME: 2024-06-25 06:04:15 -- STEP: 571/591 -- GLOBAL_STEP: 11800 + | > loss_disc: 2.750260591506958 (2.747759473720491) + | > loss_disc_real_0: 0.23288127779960632 (0.2191837050033744) + | > loss_disc_real_1: 0.21623089909553528 (0.23066508276763187) + | > loss_disc_real_2: 0.2304011434316635 (0.23491862356245205) + | > loss_disc_real_3: 0.24008968472480774 (0.23562356070232057) + | > loss_disc_real_4: 0.2638201117515564 (0.2352258035107377) + | > loss_disc_real_5: 0.2383504956960678 (0.2382652397853063) + | > loss_0: 2.750260591506958 (2.747759473720491) + | > grad_norm_0: tensor(3.6588, device='cuda:0') (tensor(10.0675, device='cuda:0')) + | > loss_gen: 1.7991535663604736 (1.866858636017647) + | > loss_kl: 2.4024055004119873 (2.5022633894103654) + | > loss_feat: 3.053755044937134 (3.127733928727187) + | > loss_mel: 21.41054344177246 (21.75033157468886) + | > loss_duration: 1.4252307415008545 (1.3890743405931671) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.091087341308594 (30.636261818079106) + | > grad_norm_1: tensor(89.4813, device='cuda:0') (tensor(131.6768, device='cuda:0')) + | > current_lr_0: 0.00019952553399667344 + | > current_lr_1: 0.00019952553399667344 + | > step_time: 1.2058 (0.9423841357022368) + | > loader_time: 0.0255 (0.01532661601666185) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036309932140593834 (-0.0001668676416924659) + | > avg_loss_disc: 2.751353760983081 (+0.05204609607128319) + | > avg_loss_disc_real_0: 0.21076418110664855 (-0.022503875988595018) + | > avg_loss_disc_real_1: 0.23909296158780444 (+0.030299676859632446) + | > avg_loss_disc_real_2: 0.23183104205638805 (-0.020574137251427815) + | > avg_loss_disc_real_3: 0.2500516347428586 (+0.04517263015534018) + | > avg_loss_disc_real_4: 0.23165594738848666 (-0.010881839280432848) + | > avg_loss_disc_real_5: 0.26378839066688053 (-0.004399691807462802) + | > avg_loss_0: 2.751353760983081 (+0.05204609607128319) + | > avg_loss_gen: 1.8410906411231833 (-0.05258944947668831) + | > avg_loss_kl: 2.621357283693679 (+0.09175036308613205) + | > avg_loss_feat: 3.000877101370629 (-0.06462846411035406) + | > avg_loss_mel: 22.138012662846993 (+0.24822076838067986) + | > avg_loss_duration: 1.3894687028641397 (-0.0031285362040742726) + | > avg_loss_1: 30.990806011443443 (+0.21962425556588627) + + + > EPOCH: 21/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 06:04:49)  + + --> TIME: 2024-06-25 06:04:55 -- STEP: 5/591 -- GLOBAL_STEP: 11825 + | > loss_disc: 2.7616360187530518 (2.7243090152740477) + | > loss_disc_real_0: 0.2990831732749939 (0.2244144856929779) + | > loss_disc_real_1: 0.2653842270374298 (0.23051553070545197) + | > loss_disc_real_2: 0.26350247859954834 (0.2304697573184967) + | > loss_disc_real_3: 0.28960683941841125 (0.24396079182624816) + | > loss_disc_real_4: 0.2538382411003113 (0.23718000948429108) + | > loss_disc_real_5: 0.22227154672145844 (0.23164283633232116) + | > loss_0: 2.7616360187530518 (2.7243090152740477) + | > grad_norm_0: tensor(15.7149, device='cuda:0') (tensor(8.7672, device='cuda:0')) + | > loss_gen: 1.8027434349060059 (1.9133218288421632) + | > loss_kl: 2.6459782123565674 (2.6955718040466308) + | > loss_feat: 3.347294807434082 (3.3848196506500243) + | > loss_mel: 21.95830535888672 (21.990261840820313) + | > loss_duration: 1.3435477018356323 (1.371176290512085) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.097869873046875 (31.355151748657228) + | > grad_norm_1: tensor(200.3234, device='cuda:0') (tensor(191.5403, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.7811 (0.7730380535125733) + | > loader_time: 0.0108 (0.009153938293457032) + + + --> TIME: 2024-06-25 06:05:16 -- STEP: 30/591 -- GLOBAL_STEP: 11850 + | > loss_disc: 2.7666454315185547 (2.778072190284729) + | > loss_disc_real_0: 0.2208695411682129 (0.2185811147093773) + | > loss_disc_real_1: 0.15534913539886475 (0.2399327725172043) + | > loss_disc_real_2: 0.20109035074710846 (0.23381868104139963) + | > loss_disc_real_3: 0.18178239464759827 (0.234577173491319) + | > loss_disc_real_4: 0.22820036113262177 (0.23678445021311442) + | > loss_disc_real_5: 0.24575228989124298 (0.23870318681001662) + | > loss_0: 2.7666454315185547 (2.778072190284729) + | > grad_norm_0: tensor(11.0302, device='cuda:0') (tensor(11.2096, device='cuda:0')) + | > loss_gen: 2.084014415740967 (1.9110655506451926) + | > loss_kl: 2.5819437503814697 (2.6586123148600254) + | > loss_feat: 3.1998865604400635 (3.320507518450419) + | > loss_mel: 21.72793197631836 (22.013514900207518) + | > loss_duration: 1.3701852560043335 (1.3505136211713156) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.96396255493164 (31.254214096069337) + | > grad_norm_1: tensor(209.1634, device='cuda:0') (tensor(189.5679, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.788 (0.8112055460611979) + | > loader_time: 0.0094 (0.012114540735880534) + + + --> TIME: 2024-06-25 06:05:36 -- STEP: 55/591 -- GLOBAL_STEP: 11875 + | > loss_disc: 2.757660388946533 (2.7813185345042832) + | > loss_disc_real_0: 0.3405207395553589 (0.22103857858614487) + | > loss_disc_real_1: 0.25352808833122253 (0.24818186109716248) + | > loss_disc_real_2: 0.2127677947282791 (0.23556751500476492) + | > loss_disc_real_3: 0.2482208013534546 (0.23712193153121255) + | > loss_disc_real_4: 0.21744301915168762 (0.23642784275791862) + | > loss_disc_real_5: 0.25189971923828125 (0.2384265672076832) + | > loss_0: 2.757660388946533 (2.7813185345042832) + | > grad_norm_0: tensor(22.1906, device='cuda:0') (tensor(11.4391, device='cuda:0')) + | > loss_gen: 1.770937204360962 (1.8967337998476896) + | > loss_kl: 2.4192769527435303 (2.6216351422396573) + | > loss_feat: 3.22017240524292 (3.2559005867351187) + | > loss_mel: 21.880878448486328 (21.981194652210583) + | > loss_duration: 1.390514850616455 (1.3617175015536223) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.681779861450195 (31.117181743275037) + | > grad_norm_1: tensor(151.6274, device='cuda:0') (tensor(176.6176, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.8295 (0.8083264220844616) + | > loader_time: 0.0109 (0.011218322407115589) + + + --> TIME: 2024-06-25 06:05:57 -- STEP: 80/591 -- GLOBAL_STEP: 11900 + | > loss_disc: 2.807344913482666 (2.773476234078408) + | > loss_disc_real_0: 0.163674995303154 (0.21921095829457043) + | > loss_disc_real_1: 0.2200237661600113 (0.24268867205828432) + | > loss_disc_real_2: 0.22330087423324585 (0.23492878284305335) + | > loss_disc_real_3: 0.19684801995754242 (0.23685758076608182) + | > loss_disc_real_4: 0.20820237696170807 (0.23590994756668807) + | > loss_disc_real_5: 0.1980152726173401 (0.23842867817729713) + | > loss_0: 2.807344913482666 (2.773476234078408) + | > grad_norm_0: tensor(13.7627, device='cuda:0') (tensor(10.0286, device='cuda:0')) + | > loss_gen: 1.8898305892944336 (1.8838587418198585) + | > loss_kl: 2.584414482116699 (2.5898227363824846) + | > loss_feat: 3.2853622436523438 (3.235115295648575) + | > loss_mel: 22.393558502197266 (21.929480814933775) + | > loss_duration: 1.4236319065093994 (1.368593376874924) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.576799392700195 (31.006870985031128) + | > grad_norm_1: tensor(125.6782, device='cuda:0') (tensor(160.3717, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.8194 (0.8157641619443894) + | > loader_time: 0.0101 (0.011020138859748837) + + + --> TIME: 2024-06-25 06:06:18 -- STEP: 105/591 -- GLOBAL_STEP: 11925 + | > loss_disc: 2.704542875289917 (2.765135942186629) + | > loss_disc_real_0: 0.27596980333328247 (0.2203242382832936) + | > loss_disc_real_1: 0.21644838154315948 (0.23901844010466625) + | > loss_disc_real_2: 0.21997499465942383 (0.23467912461076465) + | > loss_disc_real_3: 0.2521941363811493 (0.23612191109430222) + | > loss_disc_real_4: 0.25441429018974304 (0.23558623308227175) + | > loss_disc_real_5: 0.2547750473022461 (0.23870988871370044) + | > loss_0: 2.704542875289917 (2.765135942186629) + | > grad_norm_0: tensor(9.8330, device='cuda:0') (tensor(10.4388, device='cuda:0')) + | > loss_gen: 1.9344711303710938 (1.8805573735918317) + | > loss_kl: 2.4719810485839844 (2.5744496277400413) + | > loss_feat: 3.230415105819702 (3.232382683526902) + | > loss_mel: 21.7745304107666 (21.92423182896205) + | > loss_duration: 1.3806878328323364 (1.3751487890879315) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.792085647583008 (30.986770302908763) + | > grad_norm_1: tensor(200.3995, device='cuda:0') (tensor(149.9750, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.8276 (0.8199753670465378) + | > loader_time: 0.012 (0.011080124264671688) + + + --> TIME: 2024-06-25 06:06:40 -- STEP: 130/591 -- GLOBAL_STEP: 11950 + | > loss_disc: 2.7523672580718994 (2.7619205915010903) + | > loss_disc_real_0: 0.17118960618972778 (0.22000294889395053) + | > loss_disc_real_1: 0.21077367663383484 (0.2370682178781583) + | > loss_disc_real_2: 0.212248757481575 (0.23502676028471728) + | > loss_disc_real_3: 0.23365914821624756 (0.23607355929338014) + | > loss_disc_real_4: 0.25290387868881226 (0.23571617821088203) + | > loss_disc_real_5: 0.22193238139152527 (0.23855326840510735) + | > loss_0: 2.7523672580718994 (2.7619205915010903) + | > grad_norm_0: tensor(8.9138, device='cuda:0') (tensor(10.2547, device='cuda:0')) + | > loss_gen: 1.8473834991455078 (1.878874419285701) + | > loss_kl: 2.486882448196411 (2.5635225185981154) + | > loss_feat: 3.2616918087005615 (3.240841474899879) + | > loss_mel: 21.99724006652832 (21.92933025360107) + | > loss_duration: 1.3712505102157593 (1.3770194567166845) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.964448928833008 (30.98958809192364) + | > grad_norm_1: tensor(211.2619, device='cuda:0') (tensor(146.7594, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.8431 (0.8245845537919264) + | > loader_time: 0.0121 (0.011128011116614708) + + + --> TIME: 2024-06-25 06:07:02 -- STEP: 155/591 -- GLOBAL_STEP: 11975 + | > loss_disc: 2.7883267402648926 (2.7570205227021254) + | > loss_disc_real_0: 0.20652219653129578 (0.2193313564023664) + | > loss_disc_real_1: 0.2072228342294693 (0.23607856737029168) + | > loss_disc_real_2: 0.1827336847782135 (0.2350074274886039) + | > loss_disc_real_3: 0.23908984661102295 (0.23565121773750553) + | > loss_disc_real_4: 0.24750158190727234 (0.23516037050754793) + | > loss_disc_real_5: 0.2777847647666931 (0.23849593939319733) + | > loss_0: 2.7883267402648926 (2.7570205227021254) + | > grad_norm_0: tensor(4.1632, device='cuda:0') (tensor(10.0094, device='cuda:0')) + | > loss_gen: 1.8540046215057373 (1.8796339227307228) + | > loss_kl: 2.6291046142578125 (2.5540090253276198) + | > loss_feat: 3.0561294555664062 (3.238222383683728) + | > loss_mel: 21.672842025756836 (21.871607159030052) + | > loss_duration: 1.3885903358459473 (1.3752975363885205) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.600669860839844 (30.918769947175058) + | > grad_norm_1: tensor(54.3678, device='cuda:0') (tensor(142.3053, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.8824 (0.829354021626134) + | > loader_time: 0.0115 (0.011257207009100144) + + + --> TIME: 2024-06-25 06:07:24 -- STEP: 180/591 -- GLOBAL_STEP: 12000 + | > loss_disc: 2.847459316253662 (2.7568565991189753) + | > loss_disc_real_0: 0.3644142746925354 (0.2199536655512121) + | > loss_disc_real_1: 0.2095414102077484 (0.2352875745958752) + | > loss_disc_real_2: 0.23280233144760132 (0.23519270287619698) + | > loss_disc_real_3: 0.2384520173072815 (0.2360213014814589) + | > loss_disc_real_4: 0.24765874445438385 (0.235217574901051) + | > loss_disc_real_5: 0.25972655415534973 (0.23838952423797713) + | > loss_0: 2.847459316253662 (2.7568565991189753) + | > grad_norm_0: tensor(23.8360, device='cuda:0') (tensor(9.9539, device='cuda:0')) + | > loss_gen: 1.6300666332244873 (1.8770320170455508) + | > loss_kl: 2.567796230316162 (2.546986386511059) + | > loss_feat: 3.1262693405151367 (3.233622041013506) + | > loss_mel: 21.49416160583496 (21.855860731336808) + | > loss_duration: 1.3425054550170898 (1.3737041691939036) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.160797119140625 (30.88720529344347) + | > grad_norm_1: tensor(178.7744, device='cuda:0') (tensor(140.2574, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.856 (0.8339981291029188) + | > loader_time: 0.012 (0.011381978458828396) + + + --> TIME: 2024-06-25 06:07:46 -- STEP: 205/591 -- GLOBAL_STEP: 12025 + | > loss_disc: 2.7284090518951416 (2.7549235297412413) + | > loss_disc_real_0: 0.2952306866645813 (0.2198159626344355) + | > loss_disc_real_1: 0.24256929755210876 (0.23418332789002394) + | > loss_disc_real_2: 0.2450179010629654 (0.23504710924334643) + | > loss_disc_real_3: 0.20205554366111755 (0.23562955027673302) + | > loss_disc_real_4: 0.22678467631340027 (0.23499781700169167) + | > loss_disc_real_5: 0.24849367141723633 (0.23859043971794408) + | > loss_0: 2.7284090518951416 (2.7549235297412413) + | > grad_norm_0: tensor(13.9349, device='cuda:0') (tensor(10.0157, device='cuda:0')) + | > loss_gen: 1.8892982006072998 (1.8769304153395863) + | > loss_kl: 2.4039392471313477 (2.5397318084065494) + | > loss_feat: 3.1359050273895264 (3.2332910584240424) + | > loss_mel: 21.542362213134766 (21.841162174503978) + | > loss_duration: 1.3627207279205322 (1.373396396636963) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.334224700927734 (30.86451179690477) + | > grad_norm_1: tensor(57.3457, device='cuda:0') (tensor(142.8482, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.8827 (0.8387525116525045) + | > loader_time: 0.0133 (0.011571666671008599) + + + --> TIME: 2024-06-25 06:08:08 -- STEP: 230/591 -- GLOBAL_STEP: 12050 + | > loss_disc: 2.8387808799743652 (2.7537787364876793) + | > loss_disc_real_0: 0.14038211107254028 (0.21899210739394892) + | > loss_disc_real_1: 0.21654419600963593 (0.23364483616922213) + | > loss_disc_real_2: 0.27320125699043274 (0.23532733573861744) + | > loss_disc_real_3: 0.1857244074344635 (0.23518541630195536) + | > loss_disc_real_4: 0.22867566347122192 (0.23512698249972386) + | > loss_disc_real_5: 0.22063706815242767 (0.23850127328997073) + | > loss_0: 2.8387808799743652 (2.7537787364876793) + | > grad_norm_0: tensor(16.5141, device='cuda:0') (tensor(9.8507, device='cuda:0')) + | > loss_gen: 2.066593647003174 (1.8758764013000158) + | > loss_kl: 2.627073287963867 (2.533237400262251) + | > loss_feat: 3.1232380867004395 (3.2281271146691366) + | > loss_mel: 21.411062240600586 (21.825598244045093) + | > loss_duration: 1.3803455829620361 (1.3731778792712999) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.608312606811523 (30.836017011559527) + | > grad_norm_1: tensor(174.1591, device='cuda:0') (tensor(138.5144, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.9038 (0.8432149949281112) + | > loader_time: 0.017 (0.011816288077312966) + + + --> TIME: 2024-06-25 06:08:31 -- STEP: 255/591 -- GLOBAL_STEP: 12075 + | > loss_disc: 2.7798588275909424 (2.752592318665748) + | > loss_disc_real_0: 0.30847588181495667 (0.21919092249636557) + | > loss_disc_real_1: 0.24115073680877686 (0.23339671288050856) + | > loss_disc_real_2: 0.24416154623031616 (0.23539753258228302) + | > loss_disc_real_3: 0.21972951292991638 (0.2355864220974492) + | > loss_disc_real_4: 0.23450356721878052 (0.2349340564480015) + | > loss_disc_real_5: 0.2372836172580719 (0.23848305362112382) + | > loss_0: 2.7798588275909424 (2.752592318665748) + | > grad_norm_0: tensor(21.0666, device='cuda:0') (tensor(9.8542, device='cuda:0')) + | > loss_gen: 1.616089105606079 (1.873692113277959) + | > loss_kl: 2.4841694831848145 (2.5314245074403043) + | > loss_feat: 3.0485751628875732 (3.22341581325905) + | > loss_mel: 21.252010345458984 (21.79114305084827) + | > loss_duration: 1.3516252040863037 (1.3730498126908846) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.752470016479492 (30.792725290036667) + | > grad_norm_1: tensor(157.6106, device='cuda:0') (tensor(136.2297, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.9052 (0.8479709082958745) + | > loader_time: 0.0154 (0.012099468006807215) + + + --> TIME: 2024-06-25 06:08:54 -- STEP: 280/591 -- GLOBAL_STEP: 12100 + | > loss_disc: 2.7366199493408203 (2.751779512848173) + | > loss_disc_real_0: 0.19807446002960205 (0.21922306615327086) + | > loss_disc_real_1: 0.29849591851234436 (0.23292678163519928) + | > loss_disc_real_2: 0.2466035932302475 (0.23517678465162004) + | > loss_disc_real_3: 0.238101527094841 (0.2354326813348702) + | > loss_disc_real_4: 0.23532500863075256 (0.23486523149268967) + | > loss_disc_real_5: 0.24938485026359558 (0.2384861064276525) + | > loss_0: 2.7366199493408203 (2.751779512848173) + | > grad_norm_0: tensor(7.6715, device='cuda:0') (tensor(9.9302, device='cuda:0')) + | > loss_gen: 1.849035382270813 (1.8732779077121189) + | > loss_kl: 2.5500431060791016 (2.52723702447755) + | > loss_feat: 3.1347739696502686 (3.2207924459661754) + | > loss_mel: 21.086669921875 (21.77124794551304) + | > loss_duration: 1.4044660329818726 (1.3735712047134128) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.02499008178711 (30.76612651688712) + | > grad_norm_1: tensor(57.9896, device='cuda:0') (tensor(134.6149, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.9062 (0.852619329520634) + | > loader_time: 0.0156 (0.012356738533292498) + + + --> TIME: 2024-06-25 06:09:18 -- STEP: 305/591 -- GLOBAL_STEP: 12125 + | > loss_disc: 2.712416648864746 (2.7518116919720756) + | > loss_disc_real_0: 0.20213311910629272 (0.21945266772489078) + | > loss_disc_real_1: 0.23838414251804352 (0.2325018623813254) + | > loss_disc_real_2: 0.2522467076778412 (0.235031193543653) + | > loss_disc_real_3: 0.24894550442695618 (0.23568507950814044) + | > loss_disc_real_4: 0.24700096249580383 (0.23483328731333622) + | > loss_disc_real_5: 0.22862330079078674 (0.23845171825807604) + | > loss_0: 2.712416648864746 (2.7518116919720756) + | > grad_norm_0: tensor(5.3396, device='cuda:0') (tensor(10.2140, device='cuda:0')) + | > loss_gen: 1.801606297492981 (1.8739805252825628) + | > loss_kl: 2.5864017009735107 (2.5250426698903574) + | > loss_feat: 3.27236270904541 (3.220182701798736) + | > loss_mel: 22.068761825561523 (21.768616404298875) + | > loss_duration: 1.353271722793579 (1.373848324525552) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.08240509033203 (30.76167059101042) + | > grad_norm_1: tensor(141.3461, device='cuda:0') (tensor(136.1483, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.9123 (0.8577286477948795) + | > loader_time: 0.0172 (0.012591513649362038) + + + --> TIME: 2024-06-25 06:09:41 -- STEP: 330/591 -- GLOBAL_STEP: 12150 + | > loss_disc: 2.7021970748901367 (2.7517197536699696) + | > loss_disc_real_0: 0.22548004984855652 (0.219133984997417) + | > loss_disc_real_1: 0.17123252153396606 (0.23254008509896018) + | > loss_disc_real_2: 0.24785155057907104 (0.2351698245514523) + | > loss_disc_real_3: 0.23389017581939697 (0.23571036394798395) + | > loss_disc_real_4: 0.22093354165554047 (0.2348266334244699) + | > loss_disc_real_5: 0.25386929512023926 (0.23847627888123193) + | > loss_0: 2.7021970748901367 (2.7517197536699696) + | > grad_norm_0: tensor(5.1248, device='cuda:0') (tensor(10.0893, device='cuda:0')) + | > loss_gen: 1.8825769424438477 (1.8736989126060948) + | > loss_kl: 2.475520372390747 (2.5217155601039085) + | > loss_feat: 3.0068392753601074 (3.2137681758765018) + | > loss_mel: 21.523496627807617 (21.761216926574708) + | > loss_duration: 1.3774982690811157 (1.374720564755527) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.26593017578125 (30.745120106321394) + | > grad_norm_1: tensor(198.2659, device='cuda:0') (tensor(137.8121, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.9359 (0.8627044851129702) + | > loader_time: 0.0173 (0.012796604994571576) + + + --> TIME: 2024-06-25 06:10:05 -- STEP: 355/591 -- GLOBAL_STEP: 12175 + | > loss_disc: 2.7153520584106445 (2.7519034587161637) + | > loss_disc_real_0: 0.22411257028579712 (0.21894868950189) + | > loss_disc_real_1: 0.23928213119506836 (0.23294739882710955) + | > loss_disc_real_2: 0.18446795642375946 (0.23531410752887458) + | > loss_disc_real_3: 0.19954830408096313 (0.2355051167833973) + | > loss_disc_real_4: 0.2095157504081726 (0.2349510155513253) + | > loss_disc_real_5: 0.23038730025291443 (0.23843572299245377) + | > loss_0: 2.7153520584106445 (2.7519034587161637) + | > grad_norm_0: tensor(5.5959, device='cuda:0') (tensor(10.0796, device='cuda:0')) + | > loss_gen: 1.7517143487930298 (1.8727601521451709) + | > loss_kl: 2.4229135513305664 (2.520586942619003) + | > loss_feat: 3.238382577896118 (3.2056529568954253) + | > loss_mel: 21.770671844482422 (21.74319536450882) + | > loss_duration: 1.369038462638855 (1.3760799478477161) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.55272102355957 (30.718275344203896) + | > grad_norm_1: tensor(154.6930, device='cuda:0') (tensor(138.8268, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.9444 (0.8681374818506372) + | > loader_time: 0.0165 (0.013029613629193379) + + + --> TIME: 2024-06-25 06:10:30 -- STEP: 380/591 -- GLOBAL_STEP: 12200 + | > loss_disc: 2.694652557373047 (2.750595315506583) + | > loss_disc_real_0: 0.18403983116149902 (0.2189191690989231) + | > loss_disc_real_1: 0.23528596758842468 (0.2323872354861937) + | > loss_disc_real_2: 0.2040417343378067 (0.2353371796639342) + | > loss_disc_real_3: 0.21905212104320526 (0.23550152908030308) + | > loss_disc_real_4: 0.22298720479011536 (0.2348342920604505) + | > loss_disc_real_5: 0.23183780908584595 (0.2382987232976838) + | > loss_0: 2.694652557373047 (2.750595315506583) + | > grad_norm_0: tensor(6.0662, device='cuda:0') (tensor(9.9440, device='cuda:0')) + | > loss_gen: 1.951964259147644 (1.8729322841292932) + | > loss_kl: 2.3605003356933594 (2.518434178829195) + | > loss_feat: 3.318876266479492 (3.2042973549742446) + | > loss_mel: 22.019996643066406 (21.740744937093634) + | > loss_duration: 1.3601369857788086 (1.377575815665096) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.011474609375 (30.71398455971166) + | > grad_norm_1: tensor(202.5091, device='cuda:0') (tensor(140.3960, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 0.9767 (0.8736953754174079) + | > loader_time: 0.0182 (0.013275022255746947) + + + --> TIME: 2024-06-25 06:10:54 -- STEP: 405/591 -- GLOBAL_STEP: 12225 + | > loss_disc: 2.691375732421875 (2.75072307822145) + | > loss_disc_real_0: 0.25264662504196167 (0.21895992029228328) + | > loss_disc_real_1: 0.23134909570217133 (0.2321667307688866) + | > loss_disc_real_2: 0.24048106372356415 (0.2355263237967903) + | > loss_disc_real_3: 0.2505659759044647 (0.2355986835779967) + | > loss_disc_real_4: 0.2572547495365143 (0.23490243373829642) + | > loss_disc_real_5: 0.27294570207595825 (0.23839807090935883) + | > loss_0: 2.691375732421875 (2.75072307822145) + | > grad_norm_0: tensor(8.1677, device='cuda:0') (tensor(9.8019, device='cuda:0')) + | > loss_gen: 1.809293270111084 (1.8724970902925657) + | > loss_kl: 2.4752533435821533 (2.5165595490255495) + | > loss_feat: 2.9543628692626953 (3.1992154333326552) + | > loss_mel: 21.198434829711914 (21.74080632056719) + | > loss_duration: 1.3662035465240479 (1.3786173208260255) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.80354881286621 (30.707695727878146) + | > grad_norm_1: tensor(219.9543, device='cuda:0') (tensor(140.7955, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 1.0002 (0.8798392207534222) + | > loader_time: 0.0193 (0.013508488219461328) + + + --> TIME: 2024-06-25 06:11:20 -- STEP: 430/591 -- GLOBAL_STEP: 12250 + | > loss_disc: 2.7643465995788574 (2.751669591526652) + | > loss_disc_real_0: 0.23348017036914825 (0.21951923644126847) + | > loss_disc_real_1: 0.19761905074119568 (0.23204527185406795) + | > loss_disc_real_2: 0.2648780047893524 (0.2355520620942116) + | > loss_disc_real_3: 0.19320708513259888 (0.23541925168314645) + | > loss_disc_real_4: 0.23552457988262177 (0.23498812626267587) + | > loss_disc_real_5: 0.22460786998271942 (0.23833229514055473) + | > loss_0: 2.7643465995788574 (2.751669591526652) + | > grad_norm_0: tensor(5.9628, device='cuda:0') (tensor(10.0373, device='cuda:0')) + | > loss_gen: 1.779940128326416 (1.8720421383547228) + | > loss_kl: 2.4373157024383545 (2.513447356224062) + | > loss_feat: 3.0515706539154053 (3.192579107506331) + | > loss_mel: 21.354434967041016 (21.726683456953182) + | > loss_duration: 1.3777384757995605 (1.3792970579723993) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.000999450683594 (30.684049118396846) + | > grad_norm_1: tensor(103.3980, device='cuda:0') (tensor(141.5776, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 1.024 (0.8863972796950227) + | > loader_time: 0.0329 (0.013802995792655062) + + + --> TIME: 2024-06-25 06:11:46 -- STEP: 455/591 -- GLOBAL_STEP: 12275 + | > loss_disc: 2.7497048377990723 (2.7501204998938586) + | > loss_disc_real_0: 0.286354124546051 (0.2191997317167429) + | > loss_disc_real_1: 0.20169863104820251 (0.2317479039286519) + | > loss_disc_real_2: 0.237285777926445 (0.23537623518770867) + | > loss_disc_real_3: 0.25699689984321594 (0.2354638669844512) + | > loss_disc_real_4: 0.23992334306240082 (0.23484333809260483) + | > loss_disc_real_5: 0.23756623268127441 (0.23832309796259954) + | > loss_0: 2.7497048377990723 (2.7501204998938586) + | > grad_norm_0: tensor(13.5478, device='cuda:0') (tensor(9.8742, device='cuda:0')) + | > loss_gen: 1.8337737321853638 (1.8711583465010255) + | > loss_kl: 2.477330446243286 (2.5117264889098805) + | > loss_feat: 3.317800521850586 (3.186853141575069) + | > loss_mel: 21.790985107421875 (21.705419741620073) + | > loss_duration: 1.3877556324005127 (1.3802314221203995) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.807647705078125 (30.65538914020245) + | > grad_norm_1: tensor(66.9180, device='cuda:0') (tensor(140.8846, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 1.0118 (0.8934278959756367) + | > loader_time: 0.021 (0.014043778639573321) + + + --> TIME: 2024-06-25 06:12:12 -- STEP: 480/591 -- GLOBAL_STEP: 12300 + | > loss_disc: 2.7372114658355713 (2.749469778935114) + | > loss_disc_real_0: 0.20452278852462769 (0.21892484752461314) + | > loss_disc_real_1: 0.25683966279029846 (0.23167664129287005) + | > loss_disc_real_2: 0.23047424852848053 (0.23527553947642446) + | > loss_disc_real_3: 0.25181812047958374 (0.2354366526628534) + | > loss_disc_real_4: 0.2100498229265213 (0.23482558180888494) + | > loss_disc_real_5: 0.2456141710281372 (0.23831590736905733) + | > loss_0: 2.7372114658355713 (2.749469778935114) + | > grad_norm_0: tensor(4.1284, device='cuda:0') (tensor(9.8935, device='cuda:0')) + | > loss_gen: 1.8328417539596558 (1.8703752547502517) + | > loss_kl: 2.5409209728240967 (2.5105004712939283) + | > loss_feat: 3.0306811332702637 (3.1812642559409143) + | > loss_mel: 21.740787506103516 (21.69275322755178) + | > loss_duration: 1.3979003429412842 (1.3814804755151284) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.54313087463379 (30.636373658974964) + | > grad_norm_1: tensor(99.4778, device='cuda:0') (tensor(138.4179, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 1.0562 (0.9004164631168046) + | > loader_time: 0.0234 (0.014322816828886671) + + + --> TIME: 2024-06-25 06:12:40 -- STEP: 505/591 -- GLOBAL_STEP: 12325 + | > loss_disc: 2.758737564086914 (2.750364208221435) + | > loss_disc_real_0: 0.2780395448207855 (0.21881993566409197) + | > loss_disc_real_1: 0.2142268568277359 (0.23175083401769694) + | > loss_disc_real_2: 0.2539288401603699 (0.23536897927227587) + | > loss_disc_real_3: 0.2292257845401764 (0.23556914394444758) + | > loss_disc_real_4: 0.24156233668327332 (0.23506390877289346) + | > loss_disc_real_5: 0.239761620759964 (0.23839549372101773) + | > loss_0: 2.758737564086914 (2.750364208221435) + | > grad_norm_0: tensor(11.0895, device='cuda:0') (tensor(9.9369, device='cuda:0')) + | > loss_gen: 1.791731595993042 (1.8693865044282214) + | > loss_kl: 2.4454517364501953 (2.5089448933554186) + | > loss_feat: 3.102937936782837 (3.173927723063101) + | > loss_mel: 21.889781951904297 (21.674962714166924) + | > loss_duration: 1.4241464138031006 (1.3828861137427917) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.654048919677734 (30.610107916652566) + | > grad_norm_1: tensor(53.1400, device='cuda:0') (tensor(136.8358, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 1.0734 (0.9095700660554487) + | > loader_time: 0.0173 (0.014668648785883842) + + + --> TIME: 2024-06-25 06:13:09 -- STEP: 530/591 -- GLOBAL_STEP: 12350 + | > loss_disc: 2.7314906120300293 (2.7506804011902717) + | > loss_disc_real_0: 0.16172219812870026 (0.21879627266020146) + | > loss_disc_real_1: 0.19881828129291534 (0.23164747284268433) + | > loss_disc_real_2: 0.2591242492198944 (0.23546331957826078) + | > loss_disc_real_3: 0.25358954071998596 (0.2356364646891378) + | > loss_disc_real_4: 0.23913535475730896 (0.23499132453832985) + | > loss_disc_real_5: 0.2569783627986908 (0.2383921481809526) + | > loss_0: 2.7314906120300293 (2.7506804011902717) + | > grad_norm_0: tensor(17.7576, device='cuda:0') (tensor(9.9507, device='cuda:0')) + | > loss_gen: 1.948293685913086 (1.8686235319893316) + | > loss_kl: 2.5292181968688965 (2.5069155346672507) + | > loss_feat: 2.980988025665283 (3.1677245643903627) + | > loss_mel: 21.630603790283203 (21.662747419105386) + | > loss_duration: 1.4182884693145752 (1.384364163875581) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.50739288330078 (30.590375184113125) + | > grad_norm_1: tensor(177.1638, device='cuda:0') (tensor(136.1987, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 1.1447 (0.9207626711647463) + | > loader_time: 0.0265 (0.014996036043706933) + + + --> TIME: 2024-06-25 06:13:41 -- STEP: 555/591 -- GLOBAL_STEP: 12375 + | > loss_disc: 2.700988292694092 (2.7505677236093033) + | > loss_disc_real_0: 0.21209725737571716 (0.21880069146822165) + | > loss_disc_real_1: 0.25628983974456787 (0.2315739179516698) + | > loss_disc_real_2: 0.25694355368614197 (0.23546376324988705) + | > loss_disc_real_3: 0.2417813390493393 (0.23561581896769035) + | > loss_disc_real_4: 0.23728583753108978 (0.2350287857356372) + | > loss_disc_real_5: 0.21908068656921387 (0.23837814355218734) + | > loss_0: 2.700988292694092 (2.7505677236093033) + | > grad_norm_0: tensor(4.3399, device='cuda:0') (tensor(9.9223, device='cuda:0')) + | > loss_gen: 1.9300129413604736 (1.8681598665477994) + | > loss_kl: 2.49452805519104 (2.5047244875280725) + | > loss_feat: 3.1238601207733154 (3.161951843467919) + | > loss_mel: 21.644804000854492 (21.64944235655639) + | > loss_duration: 1.4580847024917603 (1.3860248522715535) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.651288986206055 (30.57030336878321) + | > grad_norm_1: tensor(165.4817, device='cuda:0') (tensor(135.1980, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 1.211 (0.9361579061628458) + | > loader_time: 0.0246 (0.015354306418616495) + + + --> TIME: 2024-06-25 06:14:13 -- STEP: 580/591 -- GLOBAL_STEP: 12400 + | > loss_disc: 2.7601029872894287 (2.751422221907252) + | > loss_disc_real_0: 0.21560069918632507 (0.2190532740847818) + | > loss_disc_real_1: 0.23051875829696655 (0.23168640003122132) + | > loss_disc_real_2: 0.23614761233329773 (0.23550386169347276) + | > loss_disc_real_3: 0.2800944447517395 (0.23563802843463832) + | > loss_disc_real_4: 0.2524468004703522 (0.23497487332286507) + | > loss_disc_real_5: 0.26045697927474976 (0.23846092879258354) + | > loss_0: 2.7601029872894287 (2.751422221907252) + | > grad_norm_0: tensor(5.0653, device='cuda:0') (tensor(10.1160, device='cuda:0')) + | > loss_gen: 1.8972400426864624 (1.867891553969219) + | > loss_kl: 2.4332213401794434 (2.502396084111314) + | > loss_feat: 2.7726829051971436 (3.155489349365234) + | > loss_mel: 21.396114349365234 (21.63601561579211) + | > loss_duration: 1.4245662689208984 (1.3878457694218083) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.923824310302734 (30.54963833710243) + | > grad_norm_1: tensor(228.6944, device='cuda:0') (tensor(134.7287, device='cuda:0')) + | > current_lr_0: 0.00019950059330492385 + | > current_lr_1: 0.00019950059330492385 + | > step_time: 1.2534 (0.9497006375214145) + | > loader_time: 0.0275 (0.015725853114292535) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036400784837438706 (+9.085269684487238e-06) + | > avg_loss_disc: 2.782551978496795 (+0.03119821751371399) + | > avg_loss_disc_real_0: 0.2272757443975895 (+0.016511563290940945) + | > avg_loss_disc_real_1: 0.2335732049764471 (-0.00551975661135734) + | > avg_loss_disc_real_2: 0.25211843245841076 (+0.02028739040202271) + | > avg_loss_disc_real_3: 0.21701860110810461 (-0.03303303363475396) + | > avg_loss_disc_real_4: 0.19886556838421113 (-0.03279037900427553) + | > avg_loss_disc_real_5: 0.20725041976634492 (-0.05653797090053561) + | > avg_loss_0: 2.782551978496795 (+0.03119821751371399) + | > avg_loss_gen: 1.7290857142590461 (-0.11200492686413721) + | > avg_loss_kl: 2.5871970653533927 (-0.034160218340286264) + | > avg_loss_feat: 3.139851615784016 (+0.13897451441338715) + | > avg_loss_mel: 21.7867548922275 (-0.3512577706194939) + | > avg_loss_duration: 1.386183094471059 (-0.0032856083930807856) + | > avg_loss_1: 30.62907243282237 (-0.3617335786210738) + + + > EPOCH: 22/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 06:14:35)  + + --> TIME: 2024-06-25 06:14:48 -- STEP: 14/591 -- GLOBAL_STEP: 12425 + | > loss_disc: 2.8203139305114746 (2.724263770239694) + | > loss_disc_real_0: 0.3194711208343506 (0.2232855302946908) + | > loss_disc_real_1: 0.21796149015426636 (0.23011524975299835) + | > loss_disc_real_2: 0.2019493281841278 (0.23392885071890696) + | > loss_disc_real_3: 0.23096810281276703 (0.2360250832779067) + | > loss_disc_real_4: 0.23137937486171722 (0.23580287290470942) + | > loss_disc_real_5: 0.21898284554481506 (0.23959742380040033) + | > loss_0: 2.8203139305114746 (2.724263770239694) + | > grad_norm_0: tensor(14.5773, device='cuda:0') (tensor(8.7332, device='cuda:0')) + | > loss_gen: 1.768445611000061 (1.9038097092083521) + | > loss_kl: 2.6637845039367676 (2.6374971526009694) + | > loss_feat: 3.362224817276001 (3.4365538699286327) + | > loss_mel: 21.56784439086914 (21.863831111363) + | > loss_duration: 1.3223836421966553 (1.351759101663317) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.684682846069336 (31.193450927734375) + | > grad_norm_1: tensor(105.8750, device='cuda:0') (tensor(164.5623, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.7883 (0.7697076456887382) + | > loader_time: 0.0097 (0.008703436170305525) + + + --> TIME: 2024-06-25 06:15:08 -- STEP: 39/591 -- GLOBAL_STEP: 12450 + | > loss_disc: 2.780280828475952 (2.7291744794601054) + | > loss_disc_real_0: 0.18445859849452972 (0.21768576690019706) + | > loss_disc_real_1: 0.22799283266067505 (0.2286638468503952) + | > loss_disc_real_2: 0.23539435863494873 (0.23305276189094934) + | > loss_disc_real_3: 0.22617317736148834 (0.23353655941975424) + | > loss_disc_real_4: 0.23597893118858337 (0.23577845707917824) + | > loss_disc_real_5: 0.24583210051059723 (0.23935814736745295) + | > loss_0: 2.780280828475952 (2.7291744794601054) + | > grad_norm_0: tensor(9.4695, device='cuda:0') (tensor(10.1310, device='cuda:0')) + | > loss_gen: 1.9989912509918213 (1.8970535229413936) + | > loss_kl: 2.5176408290863037 (2.6072382193345285) + | > loss_feat: 3.3965070247650146 (3.3593079371330066) + | > loss_mel: 22.26868438720703 (21.750083189744217) + | > loss_duration: 1.3724720478057861 (1.35556977834457) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.554296493530273 (30.969252513005184) + | > grad_norm_1: tensor(170.7017, device='cuda:0') (tensor(147.2968, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.8206 (0.785624834207388) + | > loader_time: 0.011 (0.00950485009413499) + + + --> TIME: 2024-06-25 06:15:28 -- STEP: 64/591 -- GLOBAL_STEP: 12475 + | > loss_disc: 2.705232858657837 (2.732730947434902) + | > loss_disc_real_0: 0.27508509159088135 (0.21970803965814412) + | > loss_disc_real_1: 0.26452866196632385 (0.2286570982541889) + | > loss_disc_real_2: 0.2210852950811386 (0.2335008131340146) + | > loss_disc_real_3: 0.20895735919475555 (0.235337239690125) + | > loss_disc_real_4: 0.21172326803207397 (0.23467497061938047) + | > loss_disc_real_5: 0.242705836892128 (0.23798150289803743) + | > loss_0: 2.705232858657837 (2.732730947434902) + | > grad_norm_0: tensor(10.3278, device='cuda:0') (tensor(10.3174, device='cuda:0')) + | > loss_gen: 1.771600604057312 (1.8839004281908274) + | > loss_kl: 2.4120984077453613 (2.5800860486924644) + | > loss_feat: 3.3223562240600586 (3.3240974955260754) + | > loss_mel: 22.277921676635742 (21.76501402258873) + | > loss_duration: 1.3851975202560425 (1.3648396860808134) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.169174194335938 (30.917937725782394) + | > grad_norm_1: tensor(121.8642, device='cuda:0') (tensor(147.7130, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.8025 (0.7956730350852013) + | > loader_time: 0.0103 (0.009793933480978012) + + + --> TIME: 2024-06-25 06:15:49 -- STEP: 89/591 -- GLOBAL_STEP: 12500 + | > loss_disc: 2.736438512802124 (2.7333418894349863) + | > loss_disc_real_0: 0.22452601790428162 (0.2178180893318037) + | > loss_disc_real_1: 0.24154910445213318 (0.22865859343764489) + | > loss_disc_real_2: 0.23689599335193634 (0.23376931065923712) + | > loss_disc_real_3: 0.20483112335205078 (0.23505461115515633) + | > loss_disc_real_4: 0.20645985007286072 (0.2352801842635937) + | > loss_disc_real_5: 0.21289606392383575 (0.2383557849385765) + | > loss_0: 2.736438512802124 (2.7333418894349863) + | > grad_norm_0: tensor(5.3393, device='cuda:0') (tensor(9.6756, device='cuda:0')) + | > loss_gen: 1.9182734489440918 (1.8840535852346527) + | > loss_kl: 2.382396697998047 (2.5697105011243493) + | > loss_feat: 3.298299789428711 (3.3287783419148305) + | > loss_mel: 21.55916976928711 (21.75992397779829) + | > loss_duration: 1.4272443056106567 (1.372564223375213) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.585384368896484 (30.91503070445543) + | > grad_norm_1: tensor(253.6659, device='cuda:0') (tensor(152.6692, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.8209 (0.8042474275224665) + | > loader_time: 0.0103 (0.010057564531819203) + + + --> TIME: 2024-06-25 06:16:11 -- STEP: 114/591 -- GLOBAL_STEP: 12525 + | > loss_disc: 2.711883068084717 (2.7361925777636085) + | > loss_disc_real_0: 0.23329007625579834 (0.21946103838190698) + | > loss_disc_real_1: 0.24422937631607056 (0.22905524388739937) + | > loss_disc_real_2: 0.2706128656864166 (0.23440613754485784) + | > loss_disc_real_3: 0.23971012234687805 (0.23466595066221138) + | > loss_disc_real_4: 0.23979221284389496 (0.23463948214786096) + | > loss_disc_real_5: 0.2445702850818634 (0.23746640091402488) + | > loss_0: 2.711883068084717 (2.7361925777636085) + | > grad_norm_0: tensor(6.2364, device='cuda:0') (tensor(10.4469, device='cuda:0')) + | > loss_gen: 1.9159631729125977 (1.8846374363229985) + | > loss_kl: 2.3780202865600586 (2.5551004221564835) + | > loss_feat: 3.487150192260742 (3.321726782280102) + | > loss_mel: 21.8903865814209 (21.72609724078262) + | > loss_duration: 1.4027988910675049 (1.3759546583158928) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.07431983947754 (30.86351662351374) + | > grad_norm_1: tensor(50.8482, device='cuda:0') (tensor(148.7801, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.8429 (0.811334534695274) + | > loader_time: 0.0107 (0.010242485163504613) + + + --> TIME: 2024-06-25 06:16:32 -- STEP: 139/591 -- GLOBAL_STEP: 12550 + | > loss_disc: 2.6903235912323 (2.737108590791552) + | > loss_disc_real_0: 0.2205456793308258 (0.21829671326944297) + | > loss_disc_real_1: 0.20117893815040588 (0.22927508969529928) + | > loss_disc_real_2: 0.2531363368034363 (0.23463441581606007) + | > loss_disc_real_3: 0.24098233878612518 (0.23476766811000357) + | > loss_disc_real_4: 0.2483440786600113 (0.23548666210912114) + | > loss_disc_real_5: 0.24058566987514496 (0.23795964938702344) + | > loss_0: 2.6903235912323 (2.737108590791552) + | > grad_norm_0: tensor(4.8552, device='cuda:0') (tensor(10.2037, device='cuda:0')) + | > loss_gen: 1.8089183568954468 (1.8796020806264535) + | > loss_kl: 2.467487335205078 (2.547002799219364) + | > loss_feat: 3.2762317657470703 (3.3002983477475825) + | > loss_mel: 21.991199493408203 (21.708659810127973) + | > loss_duration: 1.39052152633667 (1.3759542712204749) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.934358596801758 (30.81151738612772) + | > grad_norm_1: tensor(155.1712, device='cuda:0') (tensor(146.0526, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.8516 (0.8174737683303065) + | > loader_time: 0.0113 (0.010421933030053005) + + + --> TIME: 2024-06-25 06:16:54 -- STEP: 164/591 -- GLOBAL_STEP: 12575 + | > loss_disc: 2.7474253177642822 (2.7363208256116742) + | > loss_disc_real_0: 0.1691613793373108 (0.2172963078156477) + | > loss_disc_real_1: 0.23764227330684662 (0.22914964778394234) + | > loss_disc_real_2: 0.2096167951822281 (0.23450375893493977) + | > loss_disc_real_3: 0.24017418920993805 (0.2348906961156101) + | > loss_disc_real_4: 0.22229324281215668 (0.23508721544611744) + | > loss_disc_real_5: 0.23374412953853607 (0.23790109575521656) + | > loss_0: 2.7474253177642822 (2.7363208256116742) + | > grad_norm_0: tensor(8.9564, device='cuda:0') (tensor(10.0344, device='cuda:0')) + | > loss_gen: 1.9943606853485107 (1.8787156641483307) + | > loss_kl: 2.409986972808838 (2.542062757945642) + | > loss_feat: 3.105175495147705 (3.2939498191926524) + | > loss_mel: 21.60626983642578 (21.69178476566222) + | > loss_duration: 1.374861478805542 (1.3739072200728628) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.49065399169922 (30.78042028008438) + | > grad_norm_1: tensor(136.3457, device='cuda:0') (tensor(145.3517, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.8693 (0.8238109161214132) + | > loader_time: 0.0114 (0.010561133303293368) + + + --> TIME: 2024-06-25 06:17:17 -- STEP: 189/591 -- GLOBAL_STEP: 12600 + | > loss_disc: 2.743396520614624 (2.7363047965619938) + | > loss_disc_real_0: 0.22089581191539764 (0.21776669314771732) + | > loss_disc_real_1: 0.2027006447315216 (0.228373724474478) + | > loss_disc_real_2: 0.24194251000881195 (0.2345725944433263) + | > loss_disc_real_3: 0.20800870656967163 (0.23502341764313833) + | > loss_disc_real_4: 0.22510764002799988 (0.23487612305494843) + | > loss_disc_real_5: 0.21133089065551758 (0.23801148142764178) + | > loss_0: 2.743396520614624 (2.7363047965619938) + | > grad_norm_0: tensor(4.1216, device='cuda:0') (tensor(10.1436, device='cuda:0')) + | > loss_gen: 1.9277315139770508 (1.8791713935357552) + | > loss_kl: 2.5834710597991943 (2.54257222332021) + | > loss_feat: 3.419670581817627 (3.2985115921686554) + | > loss_mel: 21.756103515625 (21.684045519147606) + | > loss_duration: 1.3417704105377197 (1.371927606365668) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.02874755859375 (30.77622835472147) + | > grad_norm_1: tensor(114.7917, device='cuda:0') (tensor(141.1593, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.8588 (0.8306879694499668) + | > loader_time: 0.0128 (0.010778905222655604) + + + --> TIME: 2024-06-25 06:17:39 -- STEP: 214/591 -- GLOBAL_STEP: 12625 + | > loss_disc: 2.7061054706573486 (2.738957808396527) + | > loss_disc_real_0: 0.16229604184627533 (0.21814014793974218) + | > loss_disc_real_1: 0.2724384367465973 (0.2292609014243723) + | > loss_disc_real_2: 0.25598886609077454 (0.23479557023427197) + | > loss_disc_real_3: 0.2538629472255707 (0.235107394927573) + | > loss_disc_real_4: 0.19504188001155853 (0.23486519033942266) + | > loss_disc_real_5: 0.2116900235414505 (0.23792458450961335) + | > loss_0: 2.7061054706573486 (2.738957808396527) + | > grad_norm_0: tensor(14.1850, device='cuda:0') (tensor(10.0289, device='cuda:0')) + | > loss_gen: 1.9786067008972168 (1.8792247660806245) + | > loss_kl: 2.585963010787964 (2.540680379511041) + | > loss_feat: 3.3077282905578613 (3.297357987020616) + | > loss_mel: 22.18256950378418 (21.701088940985855) + | > loss_duration: 1.365548014640808 (1.3713513339791343) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.420413970947266 (30.789703404792) + | > grad_norm_1: tensor(153.1725, device='cuda:0') (tensor(138.6329, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.9192 (0.8372982506440066) + | > loader_time: 0.0138 (0.01106591870851606) + + + --> TIME: 2024-06-25 06:18:02 -- STEP: 239/591 -- GLOBAL_STEP: 12650 + | > loss_disc: 2.7499759197235107 (2.740572810671819) + | > loss_disc_real_0: 0.30521160364151 (0.21835968765005406) + | > loss_disc_real_1: 0.21230995655059814 (0.22918681282877423) + | > loss_disc_real_2: 0.23541222512722015 (0.2350957418585422) + | > loss_disc_real_3: 0.3369011878967285 (0.23548444251385692) + | > loss_disc_real_4: 0.25403109192848206 (0.235112696387279) + | > loss_disc_real_5: 0.2497553676366806 (0.2382403690695264) + | > loss_0: 2.7499759197235107 (2.740572810671819) + | > grad_norm_0: tensor(16.2567, device='cuda:0') (tensor(9.9538, device='cuda:0')) + | > loss_gen: 1.6880053281784058 (1.8770275849178746) + | > loss_kl: 2.5271103382110596 (2.5409577411587776) + | > loss_feat: 3.061764717102051 (3.2866139441853277) + | > loss_mel: 21.453983306884766 (21.71085420712268) + | > loss_duration: 1.358306646347046 (1.3709961210833435) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.089170455932617 (30.78644958400327) + | > grad_norm_1: tensor(75.4499, device='cuda:0') (tensor(134.2919, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.9396 (0.8432092626723287) + | > loader_time: 0.0141 (0.011416279621204072) + + + --> TIME: 2024-06-25 06:18:25 -- STEP: 264/591 -- GLOBAL_STEP: 12675 + | > loss_disc: 2.6827635765075684 (2.7409150744929462) + | > loss_disc_real_0: 0.26476961374282837 (0.21825651353168668) + | > loss_disc_real_1: 0.22319428622722626 (0.22931937974962321) + | > loss_disc_real_2: 0.19691191613674164 (0.2346703473930106) + | > loss_disc_real_3: 0.2614484131336212 (0.23558524764622701) + | > loss_disc_real_4: 0.25868383049964905 (0.2352501754507874) + | > loss_disc_real_5: 0.24846984446048737 (0.23835261809554967) + | > loss_0: 2.6827635765075684 (2.7409150744929462) + | > grad_norm_0: tensor(10.2435, device='cuda:0') (tensor(10.0635, device='cuda:0')) + | > loss_gen: 1.8878461122512817 (1.8756797286597164) + | > loss_kl: 2.4920461177825928 (2.5388736381675265) + | > loss_feat: 3.503736734390259 (3.276131553180289) + | > loss_mel: 21.62619400024414 (21.68030433943778) + | > loss_duration: 1.402632236480713 (1.3709495772014966) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.91245460510254 (30.741938814972386) + | > grad_norm_1: tensor(209.0192, device='cuda:0') (tensor(131.4977, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.939 (0.8505734060749864) + | > loader_time: 0.0159 (0.011710671764431578) + + + --> TIME: 2024-06-25 06:18:49 -- STEP: 289/591 -- GLOBAL_STEP: 12700 + | > loss_disc: 2.7281906604766846 (2.7408529870650344) + | > loss_disc_real_0: 0.18644800782203674 (0.2179394921990415) + | > loss_disc_real_1: 0.2502109408378601 (0.22959608437693244) + | > loss_disc_real_2: 0.22624151408672333 (0.23458575361946463) + | > loss_disc_real_3: 0.262342631816864 (0.23543188692582934) + | > loss_disc_real_4: 0.2591544985771179 (0.23513615724330955) + | > loss_disc_real_5: 0.2501223385334015 (0.23827323054566102) + | > loss_0: 2.7281906604766846 (2.7408529870650344) + | > grad_norm_0: tensor(5.8844, device='cuda:0') (tensor(10.0511, device='cuda:0')) + | > loss_gen: 1.8992705345153809 (1.875652396555178) + | > loss_kl: 2.5586423873901367 (2.5357244649972883) + | > loss_feat: 3.230740785598755 (3.272668354239017) + | > loss_mel: 21.26405143737793 (21.67987992441778) + | > loss_duration: 1.3917995691299438 (1.3716824817822468) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.344505310058594 (30.735607622404) + | > grad_norm_1: tensor(107.7277, device='cuda:0') (tensor(131.8786, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.9131 (0.8565209398632644) + | > loader_time: 0.017 (0.012009425971747272) + + + --> TIME: 2024-06-25 06:19:13 -- STEP: 314/591 -- GLOBAL_STEP: 12725 + | > loss_disc: 2.8003385066986084 (2.741967518618153) + | > loss_disc_real_0: 0.2410651445388794 (0.21825380314877083) + | > loss_disc_real_1: 0.19440606236457825 (0.22957283960785835) + | > loss_disc_real_2: 0.25886988639831543 (0.23479958647375654) + | > loss_disc_real_3: 0.2480057030916214 (0.23532753902825582) + | > loss_disc_real_4: 0.20970995724201202 (0.2349125415940953) + | > loss_disc_real_5: 0.2205226868391037 (0.23828644599694354) + | > loss_0: 2.8003385066986084 (2.741967518618153) + | > grad_norm_0: tensor(6.8598, device='cuda:0') (tensor(10.2297, device='cuda:0')) + | > loss_gen: 1.8313562870025635 (1.874682733967046) + | > loss_kl: 2.5867576599121094 (2.5332711813556164) + | > loss_feat: 3.136460304260254 (3.2661368436874083) + | > loss_mel: 21.97321319580078 (21.671046402803658) + | > loss_duration: 1.4299542903900146 (1.3718634969109946) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.957740783691406 (30.717000669734492) + | > grad_norm_1: tensor(192.4495, device='cuda:0') (tensor(131.6112, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.9317 (0.8625945285627039) + | > loader_time: 0.0162 (0.012276108097878232) + + + --> TIME: 2024-06-25 06:19:37 -- STEP: 339/591 -- GLOBAL_STEP: 12750 + | > loss_disc: 2.739114761352539 (2.741933096123306) + | > loss_disc_real_0: 0.21351632475852966 (0.21806736141003694) + | > loss_disc_real_1: 0.21933235228061676 (0.22947273115492853) + | > loss_disc_real_2: 0.24113909900188446 (0.23499339621151444) + | > loss_disc_real_3: 0.23064227402210236 (0.23534729122939715) + | > loss_disc_real_4: 0.21791912615299225 (0.23503394460080296) + | > loss_disc_real_5: 0.24602115154266357 (0.23825461832822953) + | > loss_0: 2.739114761352539 (2.741933096123306) + | > grad_norm_0: tensor(3.4340, device='cuda:0') (tensor(10.2120, device='cuda:0')) + | > loss_gen: 1.7388783693313599 (1.873779308479444) + | > loss_kl: 2.4950904846191406 (2.532281001408896) + | > loss_feat: 2.9967188835144043 (3.256083370310014) + | > loss_mel: 21.01024627685547 (21.653229648736378) + | > loss_duration: 1.406334638595581 (1.3732119098877138) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.647268295288086 (30.6885852251081) + | > grad_norm_1: tensor(51.7926, device='cuda:0') (tensor(129.3287, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.9593 (0.8696375054947396) + | > loader_time: 0.0155 (0.012503649984489149) + + + --> TIME: 2024-06-25 06:20:01 -- STEP: 364/591 -- GLOBAL_STEP: 12775 + | > loss_disc: 2.7743563652038574 (2.74147906604704) + | > loss_disc_real_0: 0.2639191448688507 (0.21794689119189653) + | > loss_disc_real_1: 0.23745664954185486 (0.2294412310552466) + | > loss_disc_real_2: 0.21294385194778442 (0.23494374530983494) + | > loss_disc_real_3: 0.2796125113964081 (0.2352249277653275) + | > loss_disc_real_4: 0.21549412608146667 (0.2351060609977979) + | > loss_disc_real_5: 0.23567871749401093 (0.2383252447778052) + | > loss_0: 2.7743563652038574 (2.74147906604704) + | > grad_norm_0: tensor(10.7384, device='cuda:0') (tensor(10.2287, device='cuda:0')) + | > loss_gen: 1.782724142074585 (1.8734414891882256) + | > loss_kl: 2.4737839698791504 (2.5307415913749547) + | > loss_feat: 3.122515916824341 (3.2493397547648493) + | > loss_mel: 21.924942016601562 (21.638428205972193) + | > loss_duration: 1.4190826416015625 (1.3746519019970524) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.72304916381836 (30.666602931179842) + | > grad_norm_1: tensor(159.3230, device='cuda:0') (tensor(127.1141, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.9825 (0.8749398955932032) + | > loader_time: 0.0174 (0.012719584035349417) + + + --> TIME: 2024-06-25 06:20:26 -- STEP: 389/591 -- GLOBAL_STEP: 12800 + | > loss_disc: 2.74472713470459 (2.74185036080662) + | > loss_disc_real_0: 0.2526008188724518 (0.21797834290790072) + | > loss_disc_real_1: 0.233878955245018 (0.22948922975357516) + | > loss_disc_real_2: 0.21960635483264923 (0.23483036975627694) + | > loss_disc_real_3: 0.2548874318599701 (0.23545573464089617) + | > loss_disc_real_4: 0.22446788847446442 (0.23496874331815076) + | > loss_disc_real_5: 0.2509317100048065 (0.23843923199605818) + | > loss_0: 2.74472713470459 (2.74185036080662) + | > grad_norm_0: tensor(9.7270, device='cuda:0') (tensor(10.2841, device='cuda:0')) + | > loss_gen: 1.8545453548431396 (1.8734093484657897) + | > loss_kl: 2.542358875274658 (2.5288853565647558) + | > loss_feat: 2.936802625656128 (3.2436833737135533) + | > loss_mel: 21.710817337036133 (21.633288322193764) + | > loss_duration: 1.3614996671676636 (1.3753499923451082) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.406024932861328 (30.65461640247649) + | > grad_norm_1: tensor(134.8965, device='cuda:0') (tensor(126.1825, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 1.0437 (0.8813929723283627) + | > loader_time: 0.018 (0.012980693410172254) + + + --> TIME: 2024-06-25 06:20:51 -- STEP: 414/591 -- GLOBAL_STEP: 12825 + | > loss_disc: 2.7569222450256348 (2.742554834499452) + | > loss_disc_real_0: 0.21224401891231537 (0.2178200349427652) + | > loss_disc_real_1: 0.22917531430721283 (0.22963703315759051) + | > loss_disc_real_2: 0.25624606013298035 (0.23489299794470053) + | > loss_disc_real_3: 0.24513256549835205 (0.23557755714597334) + | > loss_disc_real_4: 0.23895971477031708 (0.2350021224234991) + | > loss_disc_real_5: 0.24533918499946594 (0.23837729136724978) + | > loss_0: 2.7569222450256348 (2.742554834499452) + | > grad_norm_0: tensor(5.3096, device='cuda:0') (tensor(10.1815, device='cuda:0')) + | > loss_gen: 1.852722406387329 (1.8717597026179955) + | > loss_kl: 2.5664608478546143 (2.5284990930326905) + | > loss_feat: 3.073545217514038 (3.2346331503080274) + | > loss_mel: 21.327293395996094 (21.621474570122317) + | > loss_duration: 1.4319530725479126 (1.3762519754073466) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.251976013183594 (30.632618507901253) + | > grad_norm_1: tensor(135.7491, device='cuda:0') (tensor(127.2777, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 0.9819 (0.8873204338377801) + | > loader_time: 0.0179 (0.01328171854433806) + + + --> TIME: 2024-06-25 06:21:17 -- STEP: 439/591 -- GLOBAL_STEP: 12850 + | > loss_disc: 2.6870369911193848 (2.7423732530554776) + | > loss_disc_real_0: 0.16868679225444794 (0.21780851537654505) + | > loss_disc_real_1: 0.18942108750343323 (0.22945038292022393) + | > loss_disc_real_2: 0.2475343644618988 (0.23494518733241834) + | > loss_disc_real_3: 0.21650104224681854 (0.23566198647701117) + | > loss_disc_real_4: 0.24914564192295074 (0.23497868863895433) + | > loss_disc_real_5: 0.2348063588142395 (0.23844199303478206) + | > loss_0: 2.6870369911193848 (2.7423732530554776) + | > grad_norm_0: tensor(10.8357, device='cuda:0') (tensor(10.2894, device='cuda:0')) + | > loss_gen: 1.8452204465866089 (1.8719025991500646) + | > loss_kl: 2.494819402694702 (2.5263307132590613) + | > loss_feat: 3.229264259338379 (3.230863671639513) + | > loss_mel: 21.302831649780273 (21.619197823735195) + | > loss_duration: 1.4213800430297852 (1.3771571339127138) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.29351806640625 (30.62545194169655) + | > grad_norm_1: tensor(54.2552, device='cuda:0') (tensor(129.4512, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 1.0092 (0.8932701873344823) + | > loader_time: 0.0184 (0.013577021335958077) + + + --> TIME: 2024-06-25 06:21:43 -- STEP: 464/591 -- GLOBAL_STEP: 12875 + | > loss_disc: 2.754218816757202 (2.7431099671742016) + | > loss_disc_real_0: 0.2624536156654358 (0.2177914493749368) + | > loss_disc_real_1: 0.22846467792987823 (0.22978691348870253) + | > loss_disc_real_2: 0.29030710458755493 (0.2351280926758873) + | > loss_disc_real_3: 0.23561707139015198 (0.23565345976887078) + | > loss_disc_real_4: 0.25097009539604187 (0.234976623335789) + | > loss_disc_real_5: 0.25950220227241516 (0.2384729012525801) + | > loss_0: 2.754218816757202 (2.7431099671742016) + | > grad_norm_0: tensor(8.2880, device='cuda:0') (tensor(10.2701, device='cuda:0')) + | > loss_gen: 1.722795009613037 (1.8717064793253768) + | > loss_kl: 2.5139036178588867 (2.5261740617710977) + | > loss_feat: 2.9364380836486816 (3.2252323874111823) + | > loss_mel: 21.168731689453125 (21.601226305139477) + | > loss_duration: 1.43660569190979 (1.3783462669315005) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.778474807739258 (30.602685488503553) + | > grad_norm_1: tensor(72.5829, device='cuda:0') (tensor(127.9851, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 1.0195 (0.8995840523777339) + | > loader_time: 0.0215 (0.013903306476001081) + + + --> TIME: 2024-06-25 06:22:09 -- STEP: 489/591 -- GLOBAL_STEP: 12900 + | > loss_disc: 2.723845958709717 (2.7434227144791308) + | > loss_disc_real_0: 0.20689266920089722 (0.21768232741238883) + | > loss_disc_real_1: 0.25711163878440857 (0.22965698273269677) + | > loss_disc_real_2: 0.2498161792755127 (0.2353087855323936) + | > loss_disc_real_3: 0.2592625319957733 (0.23564321375575778) + | > loss_disc_real_4: 0.22166763246059418 (0.23494392450601775) + | > loss_disc_real_5: 0.2331639677286148 (0.2384676663604982) + | > loss_0: 2.723845958709717 (2.7434227144791308) + | > grad_norm_0: tensor(4.4567, device='cuda:0') (tensor(10.2155, device='cuda:0')) + | > loss_gen: 1.8663864135742188 (1.871611797980248) + | > loss_kl: 2.5860774517059326 (2.524073494967752) + | > loss_feat: 3.207927703857422 (3.2173096334032234) + | > loss_mel: 21.171855926513672 (21.58651804777742) + | > loss_duration: 1.408003568649292 (1.3796192022432818) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.240251541137695 (30.579132177590836) + | > grad_norm_1: tensor(194.1247, device='cuda:0') (tensor(128.6552, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 1.0461 (0.9071503685052167) + | > loader_time: 0.0183 (0.014196704989318224) + + + --> TIME: 2024-06-25 06:22:37 -- STEP: 514/591 -- GLOBAL_STEP: 12925 + | > loss_disc: 2.8110740184783936 (2.7453386213065127) + | > loss_disc_real_0: 0.14170923829078674 (0.2177324926458908) + | > loss_disc_real_1: 0.23168830573558807 (0.23005948514326074) + | > loss_disc_real_2: 0.23679378628730774 (0.23559967677764856) + | > loss_disc_real_3: 0.1976141631603241 (0.2355408404703734) + | > loss_disc_real_4: 0.20260795950889587 (0.23499093706273846) + | > loss_disc_real_5: 0.21962030231952667 (0.23838911478041674) + | > loss_0: 2.8110740184783936 (2.7453386213065127) + | > grad_norm_0: tensor(13.4601, device='cuda:0') (tensor(10.3274, device='cuda:0')) + | > loss_gen: 1.9539982080459595 (1.8715057463497504) + | > loss_kl: 2.5244648456573486 (2.5215929101877186) + | > loss_feat: 3.0831072330474854 (3.2106901680912943) + | > loss_mel: 21.832487106323242 (21.579836652436608) + | > loss_duration: 1.4174695014953613 (1.3810643069011224) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.811527252197266 (30.564689795794653) + | > grad_norm_1: tensor(130.1427, device='cuda:0') (tensor(130.0090, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 1.1375 (0.9164158866563197) + | > loader_time: 0.0219 (0.014471135250789182) + + + --> TIME: 2024-06-25 06:23:07 -- STEP: 539/591 -- GLOBAL_STEP: 12950 + | > loss_disc: 2.749302387237549 (2.74551509306913) + | > loss_disc_real_0: 0.28251418471336365 (0.2177603440238284) + | > loss_disc_real_1: 0.23881657421588898 (0.23011241314141342) + | > loss_disc_real_2: 0.24446013569831848 (0.23578029052667143) + | > loss_disc_real_3: 0.26892679929733276 (0.23562252842207787) + | > loss_disc_real_4: 0.21562042832374573 (0.2350915391170912) + | > loss_disc_real_5: 0.24965770542621613 (0.23845125784568752) + | > loss_0: 2.749302387237549 (2.74551509306913) + | > grad_norm_0: tensor(13.5391, device='cuda:0') (tensor(10.2710, device='cuda:0')) + | > loss_gen: 1.8831982612609863 (1.8706134787738216) + | > loss_kl: 2.4277048110961914 (2.5181054595671255) + | > loss_feat: 3.111168146133423 (3.20153953812339) + | > loss_mel: 21.34990692138672 (21.556983473570757) + | > loss_duration: 1.417412519454956 (1.3823748178959774) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.189388275146484 (30.52961676461356) + | > grad_norm_1: tensor(61.6177, device='cuda:0') (tensor(128.6009, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 1.2043 (0.9277396303824459) + | > loader_time: 0.0185 (0.01479766054808098) + + + --> TIME: 2024-06-25 06:23:39 -- STEP: 564/591 -- GLOBAL_STEP: 12975 + | > loss_disc: 2.7429697513580322 (2.745169835310456) + | > loss_disc_real_0: 0.19363167881965637 (0.21746437357249834) + | > loss_disc_real_1: 0.25521281361579895 (0.23019406886388225) + | > loss_disc_real_2: 0.23390008509159088 (0.23573254040898162) + | > loss_disc_real_3: 0.18256054818630219 (0.23557885619976843) + | > loss_disc_real_4: 0.2415236383676529 (0.23504918879756692) + | > loss_disc_real_5: 0.2390967160463333 (0.23843251163443774) + | > loss_0: 2.7429697513580322 (2.745169835310456) + | > grad_norm_0: tensor(5.5932, device='cuda:0') (tensor(10.1535, device='cuda:0')) + | > loss_gen: 1.8276642560958862 (1.8701374951406573) + | > loss_kl: 2.4096267223358154 (2.5158624091046926) + | > loss_feat: 2.998739719390869 (3.1964060495085733) + | > loss_mel: 21.02535057067871 (21.539097383512683) + | > loss_duration: 1.443578839302063 (1.3837898849172792) + | > amp_scaler: 1024.0 (512.9078014184397) + | > loss_1: 29.704957962036133 (30.505293230638436) + | > grad_norm_1: tensor(112.1078, device='cuda:0') (tensor(128.2426, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 1.1318 (0.9411241944800032) + | > loader_time: 0.0214 (0.015175892951640678) + + + --> TIME: 2024-06-25 06:24:12 -- STEP: 589/591 -- GLOBAL_STEP: 13000 + | > loss_disc: 2.684593677520752 (2.745300126197181) + | > loss_disc_real_0: 0.20224666595458984 (0.21763385719728795) + | > loss_disc_real_1: 0.23208469152450562 (0.23015437342213857) + | > loss_disc_real_2: 0.2253212332725525 (0.2357420576499152) + | > loss_disc_real_3: 0.23787978291511536 (0.23568796352538673) + | > loss_disc_real_4: 0.22559401392936707 (0.23482834976577596) + | > loss_disc_real_5: 0.22953574359416962 (0.2383238904443783) + | > loss_0: 2.684593677520752 (2.745300126197181) + | > grad_norm_0: tensor(5.2344, device='cuda:0') (tensor(10.2397, device='cuda:0')) + | > loss_gen: 1.915024995803833 (1.8701554886958798) + | > loss_kl: 2.3944387435913086 (2.512449287194348) + | > loss_feat: 3.171541452407837 (3.1910050692501595) + | > loss_mel: 21.26497459411621 (21.52318442495247) + | > loss_duration: 1.4214929342269897 (1.3856022343773182) + | > amp_scaler: 1024.0 (534.6010186757217) + | > loss_1: 30.1674747467041 (30.48239651600655) + | > grad_norm_1: tensor(61.7146, device='cuda:0') (tensor(126.5135, device='cuda:0')) + | > current_lr_0: 0.00019947565573076072 + | > current_lr_1: 0.00019947565573076072 + | > step_time: 1.2641 (0.9562839502389645) + | > loader_time: 0.0221 (0.015570133772689742) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036065375551264335 (-3.354092861743716e-05) + | > avg_loss_disc: 2.7006483331639717 (-0.08190364533282324) + | > avg_loss_disc_real_0: 0.24307942327032697 (+0.01580367887273748) + | > avg_loss_disc_real_1: 0.25793601509104386 (+0.02436281011459676) + | > avg_loss_disc_real_2: 0.23454594960872163 (-0.017572482849689125) + | > avg_loss_disc_real_3: 0.2230052925804828 (+0.0059866914723781806) + | > avg_loss_disc_real_4: 0.2372776428435711 (+0.038412074459359985) + | > avg_loss_disc_real_5: 0.22563801674132652 (+0.0183875969749816) + | > avg_loss_0: 2.7006483331639717 (-0.08190364533282324) + | > avg_loss_gen: 1.9186890860821337 (+0.18960337182308762) + | > avg_loss_kl: 2.5749591969429177 (-0.012237868410474917) + | > avg_loss_feat: 3.1198587062511036 (-0.01999290953291233) + | > avg_loss_mel: 21.077974197712347 (-0.708780694515152) + | > avg_loss_duration: 1.4102374416716559 (+0.02405434720059696) + | > avg_loss_1: 30.101718780842234 (-0.5273536519801354) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_13002.pth + + > EPOCH: 23/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 06:24:27)  + + --> TIME: 2024-06-25 06:24:47 -- STEP: 23/591 -- GLOBAL_STEP: 13025 + | > loss_disc: 2.8475735187530518 (2.7454433752142866) + | > loss_disc_real_0: 0.2155396044254303 (0.21232677801795627) + | > loss_disc_real_1: 0.26813921332359314 (0.2332642175581144) + | > loss_disc_real_2: 0.28060534596443176 (0.23575166321319083) + | > loss_disc_real_3: 0.21153007447719574 (0.2359134703874588) + | > loss_disc_real_4: 0.20747257769107819 (0.23650482815244925) + | > loss_disc_real_5: 0.2060704529285431 (0.23887558929298236) + | > loss_0: 2.8475735187530518 (2.7454433752142866) + | > grad_norm_0: tensor(5.1607, device='cuda:0') (tensor(9.1222, device='cuda:0')) + | > loss_gen: 1.9151004552841187 (1.899091362953186) + | > loss_kl: 2.828850030899048 (2.663555114165596) + | > loss_feat: 3.3607490062713623 (3.3748368698617686) + | > loss_mel: 22.171457290649414 (21.531597469163977) + | > loss_duration: 1.3372000455856323 (1.3502476733663809) + | > amp_scaler: 1024.0 (1024.0) + | > loss_1: 31.613357543945312 (30.81932847396187) + | > grad_norm_1: tensor(172.6252, device='cuda:0') (tensor(93.6312, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.8163 (0.7808071530383566) + | > loader_time: 0.0102 (0.009053437606148098) + + + --> TIME: 2024-06-25 06:25:07 -- STEP: 48/591 -- GLOBAL_STEP: 13050 + | > loss_disc: 2.8135268688201904 (2.7514412055412927) + | > loss_disc_real_0: 0.25533807277679443 (0.21501426367710033) + | > loss_disc_real_1: 0.2638164758682251 (0.23473260179162025) + | > loss_disc_real_2: 0.23957118391990662 (0.23566232652713856) + | > loss_disc_real_3: 0.23372964560985565 (0.2370443552111586) + | > loss_disc_real_4: 0.24225950241088867 (0.23547239787876606) + | > loss_disc_real_5: 0.2525380253791809 (0.23786814790219069) + | > loss_0: 2.8135268688201904 (2.7514412055412927) + | > grad_norm_0: tensor(5.8256, device='cuda:0') (tensor(9.6107, device='cuda:0')) + | > loss_gen: 1.9747531414031982 (1.8902106458942096) + | > loss_kl: 2.5793118476867676 (2.650035277009011) + | > loss_feat: 3.227224349975586 (3.3336245914300284) + | > loss_mel: 22.16310691833496 (21.638869086901348) + | > loss_duration: 1.4010183811187744 (1.3581942071517308) + | > amp_scaler: 512.0 (906.6666666666666) + | > loss_1: 31.345413208007812 (30.870933572451275) + | > grad_norm_1: tensor(262.4424, device='cuda:0') (tensor(107.0800, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.8201 (0.7896893918514252) + | > loader_time: 0.0109 (0.00972136358420054) + + + --> TIME: 2024-06-25 06:25:29 -- STEP: 73/591 -- GLOBAL_STEP: 13075 + | > loss_disc: 2.7402350902557373 (2.752111545980793) + | > loss_disc_real_0: 0.2120341956615448 (0.21638902695211645) + | > loss_disc_real_1: 0.24136808514595032 (0.233269041327581) + | > loss_disc_real_2: 0.2265339493751526 (0.23600870044264075) + | > loss_disc_real_3: 0.2671583592891693 (0.2363274979264769) + | > loss_disc_real_4: 0.24917897582054138 (0.23555133032472167) + | > loss_disc_real_5: 0.27325695753097534 (0.2380185604911961) + | > loss_0: 2.7402350902557373 (2.752111545980793) + | > grad_norm_0: tensor(3.6148, device='cuda:0') (tensor(10.3136, device='cuda:0')) + | > loss_gen: 1.8538079261779785 (1.8795858507287013) + | > loss_kl: 2.607609987258911 (2.6136442667817423) + | > loss_feat: 3.2536027431488037 (3.30205891230335) + | > loss_mel: 21.8157958984375 (21.646465954715254) + | > loss_duration: 1.3792918920516968 (1.363288843468444) + | > amp_scaler: 512.0 (771.5068493150685) + | > loss_1: 30.91010856628418 (30.805043651633067) + | > grad_norm_1: tensor(147.0091, device='cuda:0') (tensor(121.7782, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.8177 (0.8032802392358649) + | > loader_time: 0.0104 (0.010113033529830305) + + + --> TIME: 2024-06-25 06:25:50 -- STEP: 98/591 -- GLOBAL_STEP: 13100 + | > loss_disc: 2.7151167392730713 (2.744536056810496) + | > loss_disc_real_0: 0.2750329375267029 (0.21481322573155773) + | > loss_disc_real_1: 0.2349541336297989 (0.2318021094008368) + | > loss_disc_real_2: 0.2459142804145813 (0.23531118339421797) + | > loss_disc_real_3: 0.21916675567626953 (0.23569583877616998) + | > loss_disc_real_4: 0.24278096854686737 (0.23585532803316506) + | > loss_disc_real_5: 0.24419288337230682 (0.2382229101292941) + | > loss_0: 2.7151167392730713 (2.744536056810496) + | > grad_norm_0: tensor(11.9536, device='cuda:0') (tensor(9.7127, device='cuda:0')) + | > loss_gen: 1.8291077613830566 (1.8794410909925188) + | > loss_kl: 2.468707799911499 (2.5915144608945266) + | > loss_feat: 3.3512496948242188 (3.312433004379273) + | > loss_mel: 22.064424514770508 (21.669238518695433) + | > loss_duration: 1.3907039165496826 (1.3708839234040706) + | > amp_scaler: 512.0 (705.3061224489797) + | > loss_1: 31.10419273376465 (30.82351087064159) + | > grad_norm_1: tensor(81.2556, device='cuda:0') (tensor(118.1186, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.831 (0.8099817451165647) + | > loader_time: 0.0105 (0.01039305998354542) + + + --> TIME: 2024-06-25 06:26:11 -- STEP: 123/591 -- GLOBAL_STEP: 13125 + | > loss_disc: 2.7798967361450195 (2.746439086712473) + | > loss_disc_real_0: 0.17048956453800201 (0.21511495119430185) + | > loss_disc_real_1: 0.2698390781879425 (0.23137971577120992) + | > loss_disc_real_2: 0.2697582244873047 (0.2353086240165602) + | > loss_disc_real_3: 0.2939065992832184 (0.23671717917531487) + | > loss_disc_real_4: 0.22023054957389832 (0.23550083954644396) + | > loss_disc_real_5: 0.2348562479019165 (0.23877247247269484) + | > loss_0: 2.7798967361450195 (2.746439086712473) + | > grad_norm_0: tensor(10.3733, device='cuda:0') (tensor(10.0156, device='cuda:0')) + | > loss_gen: 1.8655750751495361 (1.879264576648309) + | > loss_kl: 2.6725893020629883 (2.576959922061703) + | > loss_feat: 3.3824760913848877 (3.3083518404301593) + | > loss_mel: 21.729463577270508 (21.65664886846774) + | > loss_duration: 1.3773553371429443 (1.3721696865267867) + | > amp_scaler: 512.0 (666.0162601626017) + | > loss_1: 31.0274600982666 (30.793394771048693) + | > grad_norm_1: tensor(107.5696, device='cuda:0') (tensor(121.7635, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.8474 (0.8168494410631133) + | > loader_time: 0.0116 (0.010627752397118545) + + + --> TIME: 2024-06-25 06:26:33 -- STEP: 148/591 -- GLOBAL_STEP: 13150 + | > loss_disc: 2.7929952144622803 (2.747071182405626) + | > loss_disc_real_0: 0.37635841965675354 (0.2163123119421102) + | > loss_disc_real_1: 0.23549026250839233 (0.2310042782007037) + | > loss_disc_real_2: 0.2495838701725006 (0.23564482480287552) + | > loss_disc_real_3: 0.23740878701210022 (0.235866990343139) + | > loss_disc_real_4: 0.2271619588136673 (0.23527321964502335) + | > loss_disc_real_5: 0.2292836308479309 (0.2390052795007422) + | > loss_0: 2.7929952144622803 (2.747071182405626) + | > grad_norm_0: tensor(24.1231, device='cuda:0') (tensor(10.0855, device='cuda:0')) + | > loss_gen: 1.7935984134674072 (1.8777013147199475) + | > loss_kl: 2.575188636779785 (2.5662309011897526) + | > loss_feat: 3.2344155311584473 (3.301828060601209) + | > loss_mel: 22.019046783447266 (21.65377896540873) + | > loss_duration: 1.3866454362869263 (1.371416437464791) + | > amp_scaler: 512.0 (640.0) + | > loss_1: 31.008895874023438 (30.770955588366533) + | > grad_norm_1: tensor(205.9996, device='cuda:0') (tensor(129.2660, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.851 (0.8221285230404621) + | > loader_time: 0.0112 (0.010767601631783152) + + + --> TIME: 2024-06-25 06:26:55 -- STEP: 173/591 -- GLOBAL_STEP: 13175 + | > loss_disc: 2.8007936477661133 (2.7452484458857187) + | > loss_disc_real_0: 0.2137340009212494 (0.21594289608428932) + | > loss_disc_real_1: 0.1863064020872116 (0.23021052708860076) + | > loss_disc_real_2: 0.23328566551208496 (0.23516565475160675) + | > loss_disc_real_3: 0.2367381453514099 (0.2356413527892504) + | > loss_disc_real_4: 0.20829017460346222 (0.23508599030144642) + | > loss_disc_real_5: 0.22415472567081451 (0.23898834593034204) + | > loss_0: 2.8007936477661133 (2.7452484458857187) + | > grad_norm_0: tensor(3.9488, device='cuda:0') (tensor(9.9860, device='cuda:0')) + | > loss_gen: 1.8839170932769775 (1.8767295372968464) + | > loss_kl: 2.63354754447937 (2.5563155033684883) + | > loss_feat: 3.1450788974761963 (3.2962644885730192) + | > loss_mel: 21.12659454345703 (21.633120608467586) + | > loss_duration: 1.3372851610183716 (1.3692201079660753) + | > amp_scaler: 512.0 (621.5028901734105) + | > loss_1: 30.126422882080078 (30.731650203638683) + | > grad_norm_1: tensor(151.1186, device='cuda:0') (tensor(132.4429, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.8569 (0.8277864566428124) + | > loader_time: 0.0118 (0.010886302573143403) + + + --> TIME: 2024-06-25 06:27:17 -- STEP: 198/591 -- GLOBAL_STEP: 13200 + | > loss_disc: 2.7285408973693848 (2.743132817624796) + | > loss_disc_real_0: 0.19982340931892395 (0.21596361628987573) + | > loss_disc_real_1: 0.22019924223423004 (0.23002248611113038) + | > loss_disc_real_2: 0.2444888949394226 (0.23510023680600253) + | > loss_disc_real_3: 0.29471269249916077 (0.23547186604653947) + | > loss_disc_real_4: 0.21255679428577423 (0.23488633831342062) + | > loss_disc_real_5: 0.2343684583902359 (0.23893100241518983) + | > loss_0: 2.7285408973693848 (2.743132817624796) + | > grad_norm_0: tensor(5.5110, device='cuda:0') (tensor(9.9962, device='cuda:0')) + | > loss_gen: 1.9636783599853516 (1.876934584343072) + | > loss_kl: 2.5473291873931885 (2.5542964826930654) + | > loss_feat: 3.367504596710205 (3.296852383950744) + | > loss_mel: 21.776742935180664 (21.60903668644452) + | > loss_duration: 1.3736722469329834 (1.3688915636804369) + | > amp_scaler: 512.0 (607.6767676767677) + | > loss_1: 31.028928756713867 (30.706011656558875) + | > grad_norm_1: tensor(286.8170, device='cuda:0') (tensor(129.6212, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.8641 (0.8328178964480005) + | > loader_time: 0.0115 (0.011013825734456384) + + + --> TIME: 2024-06-25 06:27:39 -- STEP: 223/591 -- GLOBAL_STEP: 13225 + | > loss_disc: 2.7363533973693848 (2.742511029735275) + | > loss_disc_real_0: 0.12611381709575653 (0.21589856130392562) + | > loss_disc_real_1: 0.20404577255249023 (0.22966286326203111) + | > loss_disc_real_2: 0.23073911666870117 (0.23516060124598276) + | > loss_disc_real_3: 0.2323959767818451 (0.23533786906789772) + | > loss_disc_real_4: 0.22333069145679474 (0.23478011880487604) + | > loss_disc_real_5: 0.22418701648712158 (0.2390818086039325) + | > loss_0: 2.7363533973693848 (2.742511029735275) + | > grad_norm_0: tensor(15.3052, device='cuda:0') (tensor(10.0704, device='cuda:0')) + | > loss_gen: 1.8897196054458618 (1.8768459184287374) + | > loss_kl: 2.4898664951324463 (2.546977038875289) + | > loss_feat: 3.248131513595581 (3.295306657996413) + | > loss_mel: 21.9993839263916 (21.614410562900147) + | > loss_duration: 1.3433562517166138 (1.3675302593162777) + | > amp_scaler: 512.0 (596.9506726457402) + | > loss_1: 30.970457077026367 (30.70107039207835) + | > grad_norm_1: tensor(68.7337, device='cuda:0') (tensor(129.1761, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.8889 (0.8377706159925247) + | > loader_time: 0.013 (0.011262528030327087) + + + --> TIME: 2024-06-25 06:28:02 -- STEP: 248/591 -- GLOBAL_STEP: 13250 + | > loss_disc: 2.796982765197754 (2.742455613228583) + | > loss_disc_real_0: 0.22185543179512024 (0.21589156823052513) + | > loss_disc_real_1: 0.21229805052280426 (0.22990628620309214) + | > loss_disc_real_2: 0.258274644613266 (0.23549690217741073) + | > loss_disc_real_3: 0.21838414669036865 (0.23535789993982162) + | > loss_disc_real_4: 0.21264810860157013 (0.23483190919843414) + | > loss_disc_real_5: 0.23203173279762268 (0.23897906927572143) + | > loss_0: 2.796982765197754 (2.742455613228583) + | > grad_norm_0: tensor(4.0145, device='cuda:0') (tensor(10.0825, device='cuda:0')) + | > loss_gen: 1.8663537502288818 (1.8784496274686628) + | > loss_kl: 2.4296743869781494 (2.5451873973492654) + | > loss_feat: 3.1544063091278076 (3.299654964477785) + | > loss_mel: 21.380369186401367 (21.62293414146669) + | > loss_duration: 1.3747771978378296 (1.3678127834873814) + | > amp_scaler: 512.0 (588.3870967741938) + | > loss_1: 30.205581665039062 (30.714038871949718) + | > grad_norm_1: tensor(207.1415, device='cuda:0') (tensor(124.3964, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.9182 (0.8428840050774236) + | > loader_time: 0.014 (0.011559003783810527) + + + --> TIME: 2024-06-25 06:28:25 -- STEP: 273/591 -- GLOBAL_STEP: 13275 + | > loss_disc: 2.678030252456665 (2.7430957743536424) + | > loss_disc_real_0: 0.17878487706184387 (0.21583619457243125) + | > loss_disc_real_1: 0.20736022293567657 (0.22978825878092657) + | > loss_disc_real_2: 0.26548314094543457 (0.2357945645248497) + | > loss_disc_real_3: 0.187416210770607 (0.23558170236510673) + | > loss_disc_real_4: 0.2276395559310913 (0.2347963104099581) + | > loss_disc_real_5: 0.21019347012043 (0.23888371431783878) + | > loss_0: 2.678030252456665 (2.7430957743536424) + | > grad_norm_0: tensor(8.7049, device='cuda:0') (tensor(9.7240, device='cuda:0')) + | > loss_gen: 2.0448312759399414 (1.8770484701617738) + | > loss_kl: 2.4491758346557617 (2.543009417397635) + | > loss_feat: 3.377793073654175 (3.290511740869655) + | > loss_mel: 21.420724868774414 (21.61676905006716) + | > loss_duration: 1.366682529449463 (1.3679035478459176) + | > amp_scaler: 512.0 (581.3919413919416) + | > loss_1: 30.659208297729492 (30.695242214552213) + | > grad_norm_1: tensor(93.3282, device='cuda:0') (tensor(122.3891, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.8869 (0.848087414717063) + | > loader_time: 0.0163 (0.011872086332831194) + + + --> TIME: 2024-06-25 06:28:48 -- STEP: 298/591 -- GLOBAL_STEP: 13300 + | > loss_disc: 2.6854946613311768 (2.744250500762223) + | > loss_disc_real_0: 0.20259714126586914 (0.2162365805632716) + | > loss_disc_real_1: 0.21579167246818542 (0.22985063323238553) + | > loss_disc_real_2: 0.2366233766078949 (0.23571821002152143) + | > loss_disc_real_3: 0.21736851334571838 (0.23549636648405317) + | > loss_disc_real_4: 0.220596581697464 (0.23485848147597088) + | > loss_disc_real_5: 0.24018648266792297 (0.2388966283542198) + | > loss_0: 2.6854946613311768 (2.744250500762223) + | > grad_norm_0: tensor(3.0683, device='cuda:0') (tensor(9.8611, device='cuda:0')) + | > loss_gen: 1.9970152378082275 (1.8765824217124276) + | > loss_kl: 2.600294828414917 (2.539662021918584) + | > loss_feat: 3.429661750793457 (3.2825262762556138) + | > loss_mel: 21.45470428466797 (21.61037742051502) + | > loss_duration: 1.353535532951355 (1.3682889906352007) + | > amp_scaler: 512.0 (575.5704697986579) + | > loss_1: 30.8352108001709 (30.677437123036224) + | > grad_norm_1: tensor(126.9271, device='cuda:0') (tensor(123.5487, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.9242 (0.853392641016301) + | > loader_time: 0.0169 (0.012179405897255713) + + + --> TIME: 2024-06-25 06:29:12 -- STEP: 323/591 -- GLOBAL_STEP: 13325 + | > loss_disc: 2.7345778942108154 (2.7427545287660777) + | > loss_disc_real_0: 0.15391919016838074 (0.21592365881145553) + | > loss_disc_real_1: 0.19064117968082428 (0.2296394386527708) + | > loss_disc_real_2: 0.2513423562049866 (0.2359102816773642) + | > loss_disc_real_3: 0.2552594840526581 (0.23560082857084716) + | > loss_disc_real_4: 0.26833102107048035 (0.2347773242476555) + | > loss_disc_real_5: 0.2794899642467499 (0.23896333229246525) + | > loss_0: 2.7345778942108154 (2.7427545287660777) + | > grad_norm_0: tensor(15.1395, device='cuda:0') (tensor(9.8166, device='cuda:0')) + | > loss_gen: 1.9103407859802246 (1.8775376893418496) + | > loss_kl: 2.5821475982666016 (2.5371581201582876) + | > loss_feat: 3.2546207904815674 (3.283794949298302) + | > loss_mel: 21.444103240966797 (21.615290691978057) + | > loss_duration: 1.3912575244903564 (1.3691601207012745) + | > amp_scaler: 512.0 (570.6501547987614) + | > loss_1: 30.58247184753418 (30.68294156667987) + | > grad_norm_1: tensor(51.4236, device='cuda:0') (tensor(124.8334, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.9312 (0.858820733640216) + | > loader_time: 0.0155 (0.012458948158996395) + + + --> TIME: 2024-06-25 06:29:36 -- STEP: 348/591 -- GLOBAL_STEP: 13350 + | > loss_disc: 2.7271249294281006 (2.743357580968704) + | > loss_disc_real_0: 0.22894416749477386 (0.2162157327964388) + | > loss_disc_real_1: 0.2454192340373993 (0.22982250556521033) + | > loss_disc_real_2: 0.18724289536476135 (0.2356655784807671) + | > loss_disc_real_3: 0.20649752020835876 (0.23559538257875662) + | > loss_disc_real_4: 0.25215476751327515 (0.2347933342145092) + | > loss_disc_real_5: 0.2365320324897766 (0.23901400980593143) + | > loss_0: 2.7271249294281006 (2.743357580968704) + | > grad_norm_0: tensor(6.4510, device='cuda:0') (tensor(9.8717, device='cuda:0')) + | > loss_gen: 1.936385154724121 (1.8762455890918603) + | > loss_kl: 2.446045398712158 (2.5336723540021087) + | > loss_feat: 3.093531608581543 (3.2752432466923493) + | > loss_mel: 21.650394439697266 (21.601895814654473) + | > loss_duration: 1.3972010612487793 (1.3704096524194747) + | > amp_scaler: 512.0 (566.4367816091949) + | > loss_1: 30.5235595703125 (30.65746663082605) + | > grad_norm_1: tensor(220.8987, device='cuda:0') (tensor(125.2028, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.9596 (0.8649256222549526) + | > loader_time: 0.016 (0.012719242052100174) + + + --> TIME: 2024-06-25 06:30:00 -- STEP: 373/591 -- GLOBAL_STEP: 13375 + | > loss_disc: 2.7382941246032715 (2.7434523572231435) + | > loss_disc_real_0: 0.16245517134666443 (0.21627593276168322) + | > loss_disc_real_1: 0.23273038864135742 (0.2298700950100978) + | > loss_disc_real_2: 0.26226335763931274 (0.2358437387457482) + | > loss_disc_real_3: 0.20513814687728882 (0.23554013917177677) + | > loss_disc_real_4: 0.20961755514144897 (0.2346840740129072) + | > loss_disc_real_5: 0.21594800055027008 (0.2388509097671381) + | > loss_0: 2.7382941246032715 (2.7434523572231435) + | > grad_norm_0: tensor(11.9840, device='cuda:0') (tensor(9.8211, device='cuda:0')) + | > loss_gen: 1.8870242834091187 (1.8757361943856004) + | > loss_kl: 2.4374892711639404 (2.5298768457714402) + | > loss_feat: 3.142886161804199 (3.271182939769754) + | > loss_mel: 20.865108489990234 (21.590844192709426) + | > loss_duration: 1.3652870655059814 (1.3718259555085417) + | > amp_scaler: 512.0 (562.788203753351) + | > loss_1: 29.697795867919922 (30.639466109288602) + | > grad_norm_1: tensor(76.3338, device='cuda:0') (tensor(124.2063, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.9574 (0.8707021382155431) + | > loader_time: 0.0155 (0.012942906678841518) + + + --> TIME: 2024-06-25 06:30:25 -- STEP: 398/591 -- GLOBAL_STEP: 13400 + | > loss_disc: 2.7149572372436523 (2.744086844238206) + | > loss_disc_real_0: 0.15167684853076935 (0.21640734477483448) + | > loss_disc_real_1: 0.2508140206336975 (0.22980792756805468) + | > loss_disc_real_2: 0.24030227959156036 (0.23605337986095468) + | > loss_disc_real_3: 0.22172732651233673 (0.23553960792832637) + | > loss_disc_real_4: 0.2456829845905304 (0.23465118100445473) + | > loss_disc_real_5: 0.26140913367271423 (0.23886061347459428) + | > loss_0: 2.7149572372436523 (2.744086844238206) + | > grad_norm_0: tensor(11.4218, device='cuda:0') (tensor(9.9372, device='cuda:0')) + | > loss_gen: 1.8103549480438232 (1.8752342303793637) + | > loss_kl: 2.556708812713623 (2.5294895483620787) + | > loss_feat: 3.11905574798584 (3.264812066926428) + | > loss_mel: 20.73064613342285 (21.581968384172455) + | > loss_duration: 1.3699649572372437 (1.3725269796261237) + | > amp_scaler: 512.0 (559.5979899497489) + | > loss_1: 29.586729049682617 (30.62403119628753) + | > grad_norm_1: tensor(50.4332, device='cuda:0') (tensor(123.0241, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.9715 (0.8772733319344832) + | > loader_time: 0.0176 (0.013207810607986837) + + + --> TIME: 2024-06-25 06:30:50 -- STEP: 423/591 -- GLOBAL_STEP: 13425 + | > loss_disc: 2.7461650371551514 (2.743894159371127) + | > loss_disc_real_0: 0.17749308049678802 (0.2161292136596168) + | > loss_disc_real_1: 0.21414995193481445 (0.22983388743642938) + | > loss_disc_real_2: 0.279680997133255 (0.2362531298511699) + | > loss_disc_real_3: 0.27405020594596863 (0.23564938263831128) + | > loss_disc_real_4: 0.19861847162246704 (0.23472401918953473) + | > loss_disc_real_5: 0.2197144478559494 (0.23871704108185238) + | > loss_0: 2.7461650371551514 (2.743894159371127) + | > grad_norm_0: tensor(9.2620, device='cuda:0') (tensor(9.7854, device='cuda:0')) + | > loss_gen: 1.8830134868621826 (1.8749864783411052) + | > loss_kl: 2.5411977767944336 (2.526980713467788) + | > loss_feat: 3.283043622970581 (3.2594142799963612) + | > loss_mel: 21.864213943481445 (21.577919569984964) + | > loss_duration: 1.4014981985092163 (1.37318044097711) + | > amp_scaler: 512.0 (556.7848699763599) + | > loss_1: 30.97296714782715 (30.61248147572186) + | > grad_norm_1: tensor(101.8938, device='cuda:0') (tensor(122.2569, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 0.9862 (0.8836706321572018) + | > loader_time: 0.0158 (0.013484900161166002) + + + --> TIME: 2024-06-25 06:31:16 -- STEP: 448/591 -- GLOBAL_STEP: 13450 + | > loss_disc: 2.7702291011810303 (2.7435000608009963) + | > loss_disc_real_0: 0.19469740986824036 (0.21621677429149194) + | > loss_disc_real_1: 0.20397743582725525 (0.22976581303269736) + | > loss_disc_real_2: 0.21476632356643677 (0.23610288852692715) + | > loss_disc_real_3: 0.2298322468996048 (0.23562505368941597) + | > loss_disc_real_4: 0.24263089895248413 (0.23462287885402994) + | > loss_disc_real_5: 0.27048081159591675 (0.23876497668347188) + | > loss_0: 2.7702291011810303 (2.7435000608009963) + | > grad_norm_0: tensor(5.1799, device='cuda:0') (tensor(9.7788, device='cuda:0')) + | > loss_gen: 1.9344747066497803 (1.87449967914394) + | > loss_kl: 2.517110586166382 (2.523882949990884) + | > loss_feat: 3.0891005992889404 (3.2546054039682653) + | > loss_mel: 21.54680633544922 (21.57295895474298) + | > loss_duration: 1.4138106107711792 (1.3743721275989498) + | > amp_scaler: 512.0 (554.2857142857149) + | > loss_1: 30.50130271911621 (30.60031912582261) + | > grad_norm_1: tensor(155.3765, device='cuda:0') (tensor(120.7063, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 1.0084 (0.8904419312519687) + | > loader_time: 0.0176 (0.013774029910564424) + + + --> TIME: 2024-06-25 06:31:42 -- STEP: 473/591 -- GLOBAL_STEP: 13475 + | > loss_disc: 2.708268642425537 (2.744296389956808) + | > loss_disc_real_0: 0.21348674595355988 (0.2163484270741521) + | > loss_disc_real_1: 0.2435348480939865 (0.2298507210316164) + | > loss_disc_real_2: 0.2219744324684143 (0.2361537785472124) + | > loss_disc_real_3: 0.22135832905769348 (0.23574680978349602) + | > loss_disc_real_4: 0.21736450493335724 (0.2346270122185318) + | > loss_disc_real_5: 0.23309020698070526 (0.23867649262097623) + | > loss_0: 2.708268642425537 (2.744296389956808) + | > grad_norm_0: tensor(5.5102, device='cuda:0') (tensor(9.8485, device='cuda:0')) + | > loss_gen: 1.8964635133743286 (1.8733852708566519) + | > loss_kl: 2.374199390411377 (2.521078482742792) + | > loss_feat: 3.236290216445923 (3.2452589417108766) + | > loss_mel: 22.232711791992188 (21.55748540301656) + | > loss_duration: 1.4086425304412842 (1.3755361350847852) + | > amp_scaler: 512.0 (552.0507399577172) + | > loss_1: 31.148305892944336 (30.572744228371104) + | > grad_norm_1: tensor(153.2433, device='cuda:0') (tensor(119.7270, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 1.0352 (0.8971298189546291) + | > loader_time: 0.021 (0.01405556106164138) + + + --> TIME: 2024-06-25 06:32:10 -- STEP: 498/591 -- GLOBAL_STEP: 13500 + | > loss_disc: 2.7485580444335938 (2.7448426638262347) + | > loss_disc_real_0: 0.23017026484012604 (0.21634446985151873) + | > loss_disc_real_1: 0.21790635585784912 (0.2299509556238431) + | > loss_disc_real_2: 0.23147420585155487 (0.2362288428717349) + | > loss_disc_real_3: 0.23737262189388275 (0.2358299824727108) + | > loss_disc_real_4: 0.2635281980037689 (0.23467957360079011) + | > loss_disc_real_5: 0.25345733761787415 (0.23872314184066282) + | > loss_0: 2.7485580444335938 (2.7448426638262347) + | > grad_norm_0: tensor(4.9449, device='cuda:0') (tensor(9.8931, device='cuda:0')) + | > loss_gen: 1.8295154571533203 (1.8730710507875468) + | > loss_kl: 2.5127575397491455 (2.5196715617275607) + | > loss_feat: 3.1941287517547607 (3.240261779731535) + | > loss_mel: 21.634593963623047 (21.545808282721957) + | > loss_duration: 1.3904625177383423 (1.3766680843380081) + | > amp_scaler: 512.0 (550.0401606425708) + | > loss_1: 30.561458587646484 (30.555480777020435) + | > grad_norm_1: tensor(101.4465, device='cuda:0') (tensor(119.0096, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 1.0808 (0.9066406076691715) + | > loader_time: 0.0211 (0.014390976074708994) + + + --> TIME: 2024-06-25 06:32:38 -- STEP: 523/591 -- GLOBAL_STEP: 13525 + | > loss_disc: 2.7273380756378174 (2.744815138512315) + | > loss_disc_real_0: 0.12999731302261353 (0.21612266976792316) + | > loss_disc_real_1: 0.19619081914424896 (0.2300218754300196) + | > loss_disc_real_2: 0.22275827825069427 (0.23617825758274838) + | > loss_disc_real_3: 0.22188885509967804 (0.23580206124322137) + | > loss_disc_real_4: 0.23143194615840912 (0.23471901820790927) + | > loss_disc_real_5: 0.216133251786232 (0.23873181478699124) + | > loss_0: 2.7273380756378174 (2.744815138512315) + | > grad_norm_0: tensor(16.2785, device='cuda:0') (tensor(9.8526, device='cuda:0')) + | > loss_gen: 1.9322179555892944 (1.872175442784968) + | > loss_kl: 2.469574213027954 (2.5171832596138817) + | > loss_feat: 3.26996111869812 (3.233787166001007) + | > loss_mel: 21.2988338470459 (21.533097268974352) + | > loss_duration: 1.4049575328826904 (1.3782841354665518) + | > amp_scaler: 512.0 (548.2217973231362) + | > loss_1: 30.37554359436035 (30.53452728811231) + | > grad_norm_1: tensor(65.1898, device='cuda:0') (tensor(118.4012, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 1.2155 (0.9160253636923621) + | > loader_time: 0.023 (0.014717711089446034) + + + --> TIME: 2024-06-25 06:33:09 -- STEP: 548/591 -- GLOBAL_STEP: 13550 + | > loss_disc: 2.7443408966064453 (2.7454720022034484) + | > loss_disc_real_0: 0.17843621969223022 (0.21638972528387598) + | > loss_disc_real_1: 0.2543383538722992 (0.23010560285544743) + | > loss_disc_real_2: 0.2367192953824997 (0.23619985324840476) + | > loss_disc_real_3: 0.21204231679439545 (0.23573632696031654) + | > loss_disc_real_4: 0.24055801331996918 (0.23477457732941123) + | > loss_disc_real_5: 0.2520919442176819 (0.23877150815551298) + | > loss_0: 2.7443408966064453 (2.7454720022034484) + | > grad_norm_0: tensor(10.0524, device='cuda:0') (tensor(9.9168, device='cuda:0')) + | > loss_gen: 1.9310312271118164 (1.8712869921739959) + | > loss_kl: 2.564406156539917 (2.516194235669434) + | > loss_feat: 3.119093894958496 (3.2261128621379815) + | > loss_mel: 21.751699447631836 (21.519702295317277) + | > loss_duration: 1.441657543182373 (1.379935706836464) + | > amp_scaler: 512.0 (546.5693430656939) + | > loss_1: 30.80788803100586 (30.513232109320423) + | > grad_norm_1: tensor(144.4232, device='cuda:0') (tensor(119.2157, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 1.2855 (0.9289053687213982) + | > loader_time: 0.021 (0.015072417520258551) + + + --> TIME: 2024-06-25 06:33:41 -- STEP: 573/591 -- GLOBAL_STEP: 13575 + | > loss_disc: 2.7375683784484863 (2.7459155477154327) + | > loss_disc_real_0: 0.1812947392463684 (0.2166418903239111) + | > loss_disc_real_1: 0.23102836310863495 (0.23001945541479202) + | > loss_disc_real_2: 0.23819179832935333 (0.23617454502491012) + | > loss_disc_real_3: 0.23633794486522675 (0.23584866086849993) + | > loss_disc_real_4: 0.2530827224254608 (0.2348323730950165) + | > loss_disc_real_5: 0.26056286692619324 (0.2388950674217617) + | > loss_0: 2.7375683784484863 (2.7459155477154327) + | > grad_norm_0: tensor(8.3375, device='cuda:0') (tensor(9.9631, device='cuda:0')) + | > loss_gen: 1.7498462200164795 (1.870635489310686) + | > loss_kl: 2.5436580181121826 (2.513392787121144) + | > loss_feat: 2.884181499481201 (3.2192606518197846) + | > loss_mel: 20.775997161865234 (21.50410015379156) + | > loss_duration: 1.400274395942688 (1.381609645189415) + | > amp_scaler: 512.0 (545.0610820244332) + | > loss_1: 29.35395622253418 (30.488998745956554) + | > grad_norm_1: tensor(63.7534, device='cuda:0') (tensor(118.9777, device='cuda:0')) + | > current_lr_0: 0.00019945072127379438 + | > current_lr_1: 0.00019945072127379438 + | > step_time: 1.2017 (0.9437952116521865) + | > loader_time: 0.0235 (0.015444871642826742) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003593617297233419 (-1.2920257893014268e-05) + | > avg_loss_disc: 2.757970246862858 (+0.057321913698886284) + | > avg_loss_disc_real_0: 0.27386302231474136 (+0.030783599044414384) + | > avg_loss_disc_real_1: 0.2428188701259329 (-0.015117144965110962) + | > avg_loss_disc_real_2: 0.26911411608787295 (+0.03456816647915131) + | > avg_loss_disc_real_3: 0.2673299429898567 (+0.04432465040937389) + | > avg_loss_disc_real_4: 0.27232108344423017 (+0.03504344060065906) + | > avg_loss_disc_real_5: 0.26372815193013943 (+0.03809013518881291) + | > avg_loss_0: 2.757970246862858 (+0.057321913698886284) + | > avg_loss_gen: 2.069447550367802 (+0.15075846428566808) + | > avg_loss_kl: 2.5256817493032897 (-0.04927744763962805) + | > avg_loss_feat: 3.1882466204622957 (+0.06838791421119206) + | > avg_loss_mel: 21.158751102204022 (+0.08077690449167463) + | > avg_loss_duration: 1.3930265117198863 (-0.017210929951769538) + | > avg_loss_1: 30.335153579711914 (+0.23343479886968055) + + + > EPOCH: 24/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 06:34:12)  + + --> TIME: 2024-06-25 06:34:19 -- STEP: 7/591 -- GLOBAL_STEP: 13600 + | > loss_disc: 2.771574020385742 (2.7634149960109164) + | > loss_disc_real_0: 0.27260884642601013 (0.21300747564860753) + | > loss_disc_real_1: 0.21535751223564148 (0.22668617963790894) + | > loss_disc_real_2: 0.21426789462566376 (0.22880075659070695) + | > loss_disc_real_3: 0.22213274240493774 (0.24116615099566324) + | > loss_disc_real_4: 0.2227245271205902 (0.2394513679402215) + | > loss_disc_real_5: 0.24109406769275665 (0.23914209646838053) + | > loss_0: 2.771574020385742 (2.7634149960109164) + | > grad_norm_0: tensor(11.8768, device='cuda:0') (tensor(15.2490, device='cuda:0')) + | > loss_gen: 1.972159743309021 (1.9044989858354842) + | > loss_kl: 2.6243536472320557 (2.7637576035090854) + | > loss_feat: 3.298154592514038 (3.3817287513187955) + | > loss_mel: 21.240966796875 (21.480812072753906) + | > loss_duration: 1.3441064357757568 (1.3612250089645386) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.4797420501709 (30.892022814069474) + | > grad_norm_1: tensor(41.1901, device='cuda:0') (tensor(103.6989, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.7819 (0.766519410269601) + | > loader_time: 0.0098 (0.008731433323451452) + + + --> TIME: 2024-06-25 06:34:39 -- STEP: 32/591 -- GLOBAL_STEP: 13625 + | > loss_disc: 2.7602834701538086 (2.7448768839240074) + | > loss_disc_real_0: 0.15415367484092712 (0.2144835467915982) + | > loss_disc_real_1: 0.26946231722831726 (0.22996992198750377) + | > loss_disc_real_2: 0.2298094481229782 (0.23316864110529423) + | > loss_disc_real_3: 0.2646898627281189 (0.2377851214259863) + | > loss_disc_real_4: 0.20712850987911224 (0.23655352042987943) + | > loss_disc_real_5: 0.24475130438804626 (0.2404605597257614) + | > loss_0: 2.7602834701538086 (2.7448768839240074) + | > grad_norm_0: tensor(13.1107, device='cuda:0') (tensor(11.0018, device='cuda:0')) + | > loss_gen: 1.7926747798919678 (1.8924813382327557) + | > loss_kl: 2.6201157569885254 (2.678843304514885) + | > loss_feat: 3.0205793380737305 (3.3758691996335983) + | > loss_mel: 21.356969833374023 (21.42550426721573) + | > loss_duration: 1.3581631183624268 (1.34539794921875) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.14850425720215 (30.71809631586075) + | > grad_norm_1: tensor(54.9865, device='cuda:0') (tensor(101.9196, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.7986 (0.781393438577652) + | > loader_time: 0.0108 (0.009473271667957306) + + + --> TIME: 2024-06-25 06:34:59 -- STEP: 57/591 -- GLOBAL_STEP: 13650 + | > loss_disc: 2.7725846767425537 (2.7437122955656887) + | > loss_disc_real_0: 0.338365375995636 (0.21778995857426994) + | > loss_disc_real_1: 0.22680653631687164 (0.22862052211635991) + | > loss_disc_real_2: 0.23984940350055695 (0.23367344888678768) + | > loss_disc_real_3: 0.25967517495155334 (0.23617239348721086) + | > loss_disc_real_4: 0.213472381234169 (0.23698736622668148) + | > loss_disc_real_5: 0.21151916682720184 (0.2394421707119858) + | > loss_0: 2.7725846767425537 (2.7437122955656887) + | > grad_norm_0: tensor(22.0418, device='cuda:0') (tensor(11.2441, device='cuda:0')) + | > loss_gen: 1.7117424011230469 (1.8903817586731493) + | > loss_kl: 2.5731353759765625 (2.637677811739738) + | > loss_feat: 3.1515963077545166 (3.3657052935215463) + | > loss_mel: 21.473546981811523 (21.522709428218374) + | > loss_duration: 1.3325203657150269 (1.3559208920127468) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.24254035949707 (30.772395251090067) + | > grad_norm_1: tensor(148.8362, device='cuda:0') (tensor(110.4815, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.8171 (0.793500540549295) + | > loader_time: 0.0101 (0.00993025512026067) + + + --> TIME: 2024-06-25 06:35:20 -- STEP: 82/591 -- GLOBAL_STEP: 13675 + | > loss_disc: 2.8500752449035645 (2.7414932658032667) + | > loss_disc_real_0: 0.09509798884391785 (0.21626369100881787) + | > loss_disc_real_1: 0.25031620264053345 (0.2290148318904202) + | > loss_disc_real_2: 0.2358555644750595 (0.2338495656121068) + | > loss_disc_real_3: 0.25738245248794556 (0.23572161466610142) + | > loss_disc_real_4: 0.25094449520111084 (0.23584999834618917) + | > loss_disc_real_5: 0.23634658753871918 (0.23895002901554108) + | > loss_0: 2.8500752449035645 (2.7414932658032667) + | > grad_norm_0: tensor(29.4315, device='cuda:0') (tensor(10.9815, device='cuda:0')) + | > loss_gen: 1.921264410018921 (1.8891957867436293) + | > loss_kl: 2.435037612915039 (2.6087315838511382) + | > loss_feat: 3.236006498336792 (3.3637932451759895) + | > loss_mel: 21.407684326171875 (21.568030264319443) + | > loss_duration: 1.3865995407104492 (1.3628178108029252) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.386592864990234 (30.79256869525444) + | > grad_norm_1: tensor(152.0950, device='cuda:0') (tensor(130.0374, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.8244 (0.8046419475136736) + | > loader_time: 0.0105 (0.01015404957096751) + + + --> TIME: 2024-06-25 06:35:42 -- STEP: 107/591 -- GLOBAL_STEP: 13700 + | > loss_disc: 2.780019760131836 (2.7403964773516782) + | > loss_disc_real_0: 0.25750666856765747 (0.21725636379462537) + | > loss_disc_real_1: 0.22564159333705902 (0.22837645107897642) + | > loss_disc_real_2: 0.26339396834373474 (0.2343007989018877) + | > loss_disc_real_3: 0.2349567860364914 (0.23534154153872874) + | > loss_disc_real_4: 0.24144357442855835 (0.23514759512705224) + | > loss_disc_real_5: 0.2420107126235962 (0.23934799613796662) + | > loss_0: 2.780019760131836 (2.7403964773516782) + | > grad_norm_0: tensor(8.6251, device='cuda:0') (tensor(11.0095, device='cuda:0')) + | > loss_gen: 1.9841196537017822 (1.889442121871164) + | > loss_kl: 2.396099328994751 (2.582435634648688) + | > loss_feat: 3.254014730453491 (3.365615804618764) + | > loss_mel: 21.38600730895996 (21.576650904717848) + | > loss_duration: 1.3543412685394287 (1.3685576826612529) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.374582290649414 (30.782702151860033) + | > grad_norm_1: tensor(153.8363, device='cuda:0') (tensor(130.9858, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.8344 (0.8123659909328569) + | > loader_time: 0.0117 (0.010420006012248104) + + + --> TIME: 2024-06-25 06:36:03 -- STEP: 132/591 -- GLOBAL_STEP: 13725 + | > loss_disc: 2.7692930698394775 (2.737360267928152) + | > loss_disc_real_0: 0.3150317668914795 (0.21708477858566877) + | > loss_disc_real_1: 0.25836578011512756 (0.22845204513181339) + | > loss_disc_real_2: 0.24978257715702057 (0.23427654644756607) + | > loss_disc_real_3: 0.23453235626220703 (0.23548434099013155) + | > loss_disc_real_4: 0.24526339769363403 (0.23514182775309592) + | > loss_disc_real_5: 0.2562655806541443 (0.239141962180535) + | > loss_0: 2.7692930698394775 (2.737360267928152) + | > grad_norm_0: tensor(19.0428, device='cuda:0') (tensor(10.4404, device='cuda:0')) + | > loss_gen: 1.6567111015319824 (1.8884126110510393) + | > loss_kl: 2.4697375297546387 (2.572279223890015) + | > loss_feat: 3.415280342102051 (3.3658681909243264) + | > loss_mel: 21.404579162597656 (21.59201736161203) + | > loss_duration: 1.3525962829589844 (1.369915497122389) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.298904418945312 (30.78849289634011) + | > grad_norm_1: tensor(46.4420, device='cuda:0') (tensor(128.4371, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.8372 (0.8187040000250846) + | > loader_time: 0.0109 (0.010569489363468056) + + + --> TIME: 2024-06-25 06:36:25 -- STEP: 157/591 -- GLOBAL_STEP: 13750 + | > loss_disc: 2.813833713531494 (2.736036337105332) + | > loss_disc_real_0: 0.19088193774223328 (0.2162086839793594) + | > loss_disc_real_1: 0.29841265082359314 (0.22898819235859402) + | > loss_disc_real_2: 0.25918835401535034 (0.2343822207989966) + | > loss_disc_real_3: 0.2840036153793335 (0.235405267993356) + | > loss_disc_real_4: 0.2378864586353302 (0.2341408095542033) + | > loss_disc_real_5: 0.22754432260990143 (0.23817984285248314) + | > loss_0: 2.813833713531494 (2.736036337105332) + | > grad_norm_0: tensor(7.5057, device='cuda:0') (tensor(10.3643, device='cuda:0')) + | > loss_gen: 1.978306531906128 (1.8899762273594072) + | > loss_kl: 2.5635600090026855 (2.5652521537367696) + | > loss_feat: 3.366658926010132 (3.360349889014177) + | > loss_mel: 21.766944885253906 (21.55712266180926) + | > loss_duration: 1.3315258026123047 (1.3679534720767075) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.006994247436523 (30.74065431971459) + | > grad_norm_1: tensor(99.0739, device='cuda:0') (tensor(127.5367, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.8609 (0.8249639052494316) + | > loader_time: 0.0114 (0.010730450320395692) + + + --> TIME: 2024-06-25 06:36:47 -- STEP: 182/591 -- GLOBAL_STEP: 13775 + | > loss_disc: 2.758115768432617 (2.7365476652816096) + | > loss_disc_real_0: 0.23139071464538574 (0.21647796415529408) + | > loss_disc_real_1: 0.2373935431241989 (0.22860224528627082) + | > loss_disc_real_2: 0.22816476225852966 (0.2346335231930345) + | > loss_disc_real_3: 0.21781352162361145 (0.23501854627342014) + | > loss_disc_real_4: 0.20970764756202698 (0.234323243027205) + | > loss_disc_real_5: 0.23673442006111145 (0.238324234855699) + | > loss_0: 2.758115768432617 (2.7365476652816096) + | > grad_norm_0: tensor(4.5400, device='cuda:0') (tensor(10.3395, device='cuda:0')) + | > loss_gen: 1.8487945795059204 (1.8884834431029938) + | > loss_kl: 2.4425368309020996 (2.557955546693487) + | > loss_feat: 3.3301196098327637 (3.359482315870432) + | > loss_mel: 21.929012298583984 (21.56372487414015) + | > loss_duration: 1.351238489151001 (1.3659134146931406) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.901700973510742 (30.735559557820416) + | > grad_norm_1: tensor(86.8751, device='cuda:0') (tensor(123.9116, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.8535 (0.8310119398347624) + | > loader_time: 0.0116 (0.010940166620107794) + + + --> TIME: 2024-06-25 06:37:10 -- STEP: 207/591 -- GLOBAL_STEP: 13800 + | > loss_disc: 2.757913112640381 (2.738658668914278) + | > loss_disc_real_0: 0.17552867531776428 (0.2160663803058546) + | > loss_disc_real_1: 0.23410192131996155 (0.22899267646137644) + | > loss_disc_real_2: 0.2590566873550415 (0.23572819042896878) + | > loss_disc_real_3: 0.2795122563838959 (0.2355028074864604) + | > loss_disc_real_4: 0.22984395921230316 (0.23445847695288452) + | > loss_disc_real_5: 0.23825812339782715 (0.23826294374350765) + | > loss_0: 2.757913112640381 (2.738658668914278) + | > grad_norm_0: tensor(10.2090, device='cuda:0') (tensor(10.5356, device='cuda:0')) + | > loss_gen: 1.7591814994812012 (1.8863043589292516) + | > loss_kl: 2.5896759033203125 (2.553915674559735) + | > loss_feat: 3.047358751296997 (3.3474144440342264) + | > loss_mel: 21.265419006347656 (21.5506185632973) + | > loss_duration: 1.3963267803192139 (1.3653971832155605) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.05796241760254 (30.70365019351388) + | > grad_norm_1: tensor(54.0964, device='cuda:0') (tensor(119.0814, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.8829 (0.8364928321561952) + | > loader_time: 0.0157 (0.01120234572369119) + + + --> TIME: 2024-06-25 06:37:32 -- STEP: 232/591 -- GLOBAL_STEP: 13825 + | > loss_disc: 2.6356875896453857 (2.7388458919936203) + | > loss_disc_real_0: 0.17982693016529083 (0.21596151982145062) + | > loss_disc_real_1: 0.2160898596048355 (0.2290083723592347) + | > loss_disc_real_2: 0.2341480702161789 (0.23585265534448213) + | > loss_disc_real_3: 0.22963185608386993 (0.23529789744523064) + | > loss_disc_real_4: 0.24591079354286194 (0.23448272113656177) + | > loss_disc_real_5: 0.2278083711862564 (0.23829542575725193) + | > loss_0: 2.6356875896453857 (2.7388458919936203) + | > grad_norm_0: tensor(5.9979, device='cuda:0') (tensor(10.2823, device='cuda:0')) + | > loss_gen: 2.0661044120788574 (1.8855557714042992) + | > loss_kl: 2.26328182220459 (2.5505643733616523) + | > loss_feat: 3.427967071533203 (3.34411732698309) + | > loss_mel: 21.047086715698242 (21.553041630777827) + | > loss_duration: 1.389836311340332 (1.3650815235129719) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.19427490234375 (30.69836059932051) + | > grad_norm_1: tensor(80.1283, device='cuda:0') (tensor(117.4956, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.8741 (0.8411572298099255) + | > loader_time: 0.0155 (0.01152756912954922) + + + --> TIME: 2024-06-25 06:37:55 -- STEP: 257/591 -- GLOBAL_STEP: 13850 + | > loss_disc: 2.6943578720092773 (2.7389782987216096) + | > loss_disc_real_0: 0.15354712307453156 (0.21543648209901173) + | > loss_disc_real_1: 0.21621274948120117 (0.22902142282589863) + | > loss_disc_real_2: 0.2696189880371094 (0.23612261264936468) + | > loss_disc_real_3: 0.24225634336471558 (0.23539782921395877) + | > loss_disc_real_4: 0.24293456971645355 (0.2345242894577145) + | > loss_disc_real_5: 0.2323557734489441 (0.23836691886302563) + | > loss_0: 2.6943578720092773 (2.7389782987216096) + | > grad_norm_0: tensor(12.2485, device='cuda:0') (tensor(10.2274, device='cuda:0')) + | > loss_gen: 2.0245625972747803 (1.8847124353920903) + | > loss_kl: 2.4626474380493164 (2.546268074428988) + | > loss_feat: 3.340549945831299 (3.339011666375839) + | > loss_mel: 20.984737396240234 (21.538072860658406) + | > loss_duration: 1.3756392002105713 (1.365235548075071) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.18813705444336 (30.673300576117253) + | > grad_norm_1: tensor(164.7148, device='cuda:0') (tensor(118.4405, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.9048 (0.8464509578066577) + | > loader_time: 0.0156 (0.011793827732249454) + + + --> TIME: 2024-06-25 06:38:18 -- STEP: 282/591 -- GLOBAL_STEP: 13875 + | > loss_disc: 2.6802167892456055 (2.7389142682366328) + | > loss_disc_real_0: 0.21512460708618164 (0.2151399734859348) + | > loss_disc_real_1: 0.19994960725307465 (0.22897258707394832) + | > loss_disc_real_2: 0.20423175394535065 (0.2360304943318908) + | > loss_disc_real_3: 0.1824188083410263 (0.23532800984086719) + | > loss_disc_real_4: 0.21551664173603058 (0.23476248926727483) + | > loss_disc_real_5: 0.23114635050296783 (0.2384249988599872) + | > loss_0: 2.6802167892456055 (2.7389142682366328) + | > grad_norm_0: tensor(4.3816, device='cuda:0') (tensor(10.1693, device='cuda:0')) + | > loss_gen: 1.9266538619995117 (1.8832491953322228) + | > loss_kl: 2.5936760902404785 (2.5439394636357076) + | > loss_feat: 3.341986656188965 (3.3340053330076507) + | > loss_mel: 21.229015350341797 (21.523756176021937) + | > loss_duration: 1.383764386177063 (1.3659214068811836) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.475095748901367 (30.650871574455966) + | > grad_norm_1: tensor(46.7253, device='cuda:0') (tensor(116.9659, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.897 (0.8512613933982579) + | > loader_time: 0.0137 (0.012054742650782806) + + + --> TIME: 2024-06-25 06:38:41 -- STEP: 307/591 -- GLOBAL_STEP: 13900 + | > loss_disc: 2.687523603439331 (2.7381967077037794) + | > loss_disc_real_0: 0.21651799976825714 (0.21495416471643633) + | > loss_disc_real_1: 0.17874298989772797 (0.2291701228785592) + | > loss_disc_real_2: 0.22945381700992584 (0.23616715872520733) + | > loss_disc_real_3: 0.2135700136423111 (0.23530923985308855) + | > loss_disc_real_4: 0.20400796830654144 (0.23450796160519316) + | > loss_disc_real_5: 0.21852713823318481 (0.23812016140948677) + | > loss_0: 2.687523603439331 (2.7381967077037794) + | > grad_norm_0: tensor(8.0490, device='cuda:0') (tensor(10.2117, device='cuda:0')) + | > loss_gen: 1.9416288137435913 (1.8835281102587424) + | > loss_kl: 2.589595079421997 (2.5442200118633354) + | > loss_feat: 3.4342963695526123 (3.330954953979592) + | > loss_mel: 20.615558624267578 (21.508107318940315) + | > loss_duration: 1.3852341175079346 (1.3663114977193582) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.9663143157959 (30.633121894314545) + | > grad_norm_1: tensor(97.8073, device='cuda:0') (tensor(117.2843, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.9369 (0.8566993346820049) + | > loader_time: 0.0163 (0.012293456820012691) + + + --> TIME: 2024-06-25 06:39:05 -- STEP: 332/591 -- GLOBAL_STEP: 13925 + | > loss_disc: 2.6935064792633057 (2.739743311003029) + | > loss_disc_real_0: 0.12690088152885437 (0.21521722529846501) + | > loss_disc_real_1: 0.25160813331604004 (0.22917023808302645) + | > loss_disc_real_2: 0.23172102868556976 (0.2363917703729078) + | > loss_disc_real_3: 0.24357819557189941 (0.23569293284272574) + | > loss_disc_real_4: 0.2684226632118225 (0.23461914304868284) + | > loss_disc_real_5: 0.26080045104026794 (0.23817581274961852) + | > loss_0: 2.6935064792633057 (2.739743311003029) + | > grad_norm_0: tensor(13.5274, device='cuda:0') (tensor(10.5064, device='cuda:0')) + | > loss_gen: 1.821712851524353 (1.8826055296932358) + | > loss_kl: 2.4872677326202393 (2.544443019183285) + | > loss_feat: 3.216886520385742 (3.3253712977271483) + | > loss_mel: 21.841196060180664 (21.511218042258772) + | > loss_duration: 1.3876310586929321 (1.3674568063523393) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.75469207763672 (30.631094685520033) + | > grad_norm_1: tensor(106.2722, device='cuda:0') (tensor(120.4886, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.9184 (0.8618625352181584) + | > loader_time: 0.0139 (0.01255193316792867) + + + --> TIME: 2024-06-25 06:39:29 -- STEP: 357/591 -- GLOBAL_STEP: 13950 + | > loss_disc: 2.68617582321167 (2.7397962498063797) + | > loss_disc_real_0: 0.17754776775836945 (0.21498700049744934) + | > loss_disc_real_1: 0.23170623183250427 (0.22927611602955503) + | > loss_disc_real_2: 0.24600747227668762 (0.23647661390257815) + | > loss_disc_real_3: 0.23764614760875702 (0.23574434195746893) + | > loss_disc_real_4: 0.2223447561264038 (0.2347746070693521) + | > loss_disc_real_5: 0.20935675501823425 (0.23821493457345402) + | > loss_0: 2.68617582321167 (2.7397962498063797) + | > grad_norm_0: tensor(12.2937, device='cuda:0') (tensor(10.1840, device='cuda:0')) + | > loss_gen: 1.9010993242263794 (1.8814777005620364) + | > loss_kl: 2.5061230659484863 (2.540640911802191) + | > loss_feat: 3.307999849319458 (3.318352272530564) + | > loss_mel: 20.918432235717773 (21.49836590977945) + | > loss_duration: 1.4198647737503052 (1.3687401024901233) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.05352020263672 (30.607576883139732) + | > grad_norm_1: tensor(62.7746, device='cuda:0') (tensor(119.5942, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.9438 (0.8673442999521891) + | > loader_time: 0.0161 (0.012774721247141432) + + + --> TIME: 2024-06-25 06:39:53 -- STEP: 382/591 -- GLOBAL_STEP: 13975 + | > loss_disc: 2.669778823852539 (2.7386952095630908) + | > loss_disc_real_0: 0.26195377111434937 (0.21503321184771848) + | > loss_disc_real_1: 0.24290555715560913 (0.22914843421175837) + | > loss_disc_real_2: 0.2604483962059021 (0.23643228622319187) + | > loss_disc_real_3: 0.27628016471862793 (0.23570438333986943) + | > loss_disc_real_4: 0.23777103424072266 (0.23461913165309667) + | > loss_disc_real_5: 0.24869289994239807 (0.23809999442069318) + | > loss_0: 2.669778823852539 (2.7386952095630908) + | > grad_norm_0: tensor(9.1505, device='cuda:0') (tensor(10.1956, device='cuda:0')) + | > loss_gen: 1.8323028087615967 (1.8818638337220197) + | > loss_kl: 2.6708171367645264 (2.5390906252786145) + | > loss_feat: 3.550236701965332 (3.3171932335299346) + | > loss_mel: 21.821876525878906 (21.494016517519338) + | > loss_duration: 1.3686565160751343 (1.3702411698421264) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.24388885498047 (30.60240537833169) + | > grad_norm_1: tensor(144.1269, device='cuda:0') (tensor(121.6266, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.9712 (0.8731204408625658) + | > loader_time: 0.0151 (0.012997288978536713) + + + --> TIME: 2024-06-25 06:40:18 -- STEP: 407/591 -- GLOBAL_STEP: 14000 + | > loss_disc: 2.7143099308013916 (2.7393787030213) + | > loss_disc_real_0: 0.1907099187374115 (0.21495112806046038) + | > loss_disc_real_1: 0.2446064054965973 (0.2291625779149573) + | > loss_disc_real_2: 0.222295880317688 (0.2363317690844618) + | > loss_disc_real_3: 0.23036456108093262 (0.23565385082429985) + | > loss_disc_real_4: 0.22941839694976807 (0.23478204429296076) + | > loss_disc_real_5: 0.23142525553703308 (0.23816578456312784) + | > loss_0: 2.7143099308013916 (2.7393787030213) + | > grad_norm_0: tensor(5.4559, device='cuda:0') (tensor(10.3023, device='cuda:0')) + | > loss_gen: 2.00250244140625 (1.880511313457161) + | > loss_kl: 2.5002806186676025 (2.5381925738707234) + | > loss_feat: 3.212109327316284 (3.3080152914623664) + | > loss_mel: 21.443689346313477 (21.484793557581806) + | > loss_duration: 1.3401319980621338 (1.3709891038679087) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.49871253967285 (30.582501840239953) + | > grad_norm_1: tensor(123.3687, device='cuda:0') (tensor(120.3871, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 0.9724 (0.8795980241433409) + | > loader_time: 0.0192 (0.013260353988160077) + + + --> TIME: 2024-06-25 06:40:44 -- STEP: 432/591 -- GLOBAL_STEP: 14025 + | > loss_disc: 2.805464267730713 (2.739339777716884) + | > loss_disc_real_0: 0.21223416924476624 (0.2149685223552364) + | > loss_disc_real_1: 0.2339695692062378 (0.22924278231544623) + | > loss_disc_real_2: 0.23247197270393372 (0.23618182788292566) + | > loss_disc_real_3: 0.25079259276390076 (0.2356988775578362) + | > loss_disc_real_4: 0.23319564759731293 (0.23472194663352436) + | > loss_disc_real_5: 0.2567054331302643 (0.23826619310097563) + | > loss_0: 2.805464267730713 (2.739339777716884) + | > grad_norm_0: tensor(5.6416, device='cuda:0') (tensor(10.1942, device='cuda:0')) + | > loss_gen: 1.7241131067276 (1.8790012780163023) + | > loss_kl: 2.447252035140991 (2.5350608991252064) + | > loss_feat: 3.0718798637390137 (3.3023741664709867) + | > loss_mel: 20.37596893310547 (21.467442861309767) + | > loss_duration: 1.3489062786102295 (1.3717419002895008) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 28.96811866760254 (30.555621098589015) + | > grad_norm_1: tensor(81.0430, device='cuda:0') (tensor(119.4139, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 1.0037 (0.8856884097611463) + | > loader_time: 0.0161 (0.01352986693382263) + + + --> TIME: 2024-06-25 06:41:09 -- STEP: 457/591 -- GLOBAL_STEP: 14050 + | > loss_disc: 2.713390588760376 (2.740346064974495) + | > loss_disc_real_0: 0.21827419102191925 (0.21486885470686648) + | > loss_disc_real_1: 0.22704121470451355 (0.22937744640677968) + | > loss_disc_real_2: 0.25395962595939636 (0.23622682416725993) + | > loss_disc_real_3: 0.19761230051517487 (0.2358846486397332) + | > loss_disc_real_4: 0.2527752220630646 (0.23480808760978983) + | > loss_disc_real_5: 0.24663233757019043 (0.23833114538072758) + | > loss_0: 2.713390588760376 (2.740346064974495) + | > grad_norm_0: tensor(5.6477, device='cuda:0') (tensor(10.2053, device='cuda:0')) + | > loss_gen: 1.875180721282959 (1.8781622943001712) + | > loss_kl: 2.421090602874756 (2.5303844325652016) + | > loss_feat: 3.0395588874816895 (3.2933178756638744) + | > loss_mel: 21.127195358276367 (21.449681113011476) + | > loss_duration: 1.360811710357666 (1.3728403442537243) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.823837280273438 (30.52438605744677) + | > grad_norm_1: tensor(153.1554, device='cuda:0') (tensor(118.6873, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 1.012 (0.8923749438521638) + | > loader_time: 0.0194 (0.013833735651729923) + + + --> TIME: 2024-06-25 06:41:36 -- STEP: 482/591 -- GLOBAL_STEP: 14075 + | > loss_disc: 2.7531850337982178 (2.740898463241292) + | > loss_disc_real_0: 0.16814790666103363 (0.215049766282323) + | > loss_disc_real_1: 0.22758238017559052 (0.22936253039421378) + | > loss_disc_real_2: 0.23494330048561096 (0.2361579812537585) + | > loss_disc_real_3: 0.2097705751657486 (0.23586009009992434) + | > loss_disc_real_4: 0.24040208756923676 (0.2349205395801928) + | > loss_disc_real_5: 0.2469586431980133 (0.23830029513702353) + | > loss_0: 2.7531850337982178 (2.740898463241292) + | > grad_norm_0: tensor(7.1144, device='cuda:0') (tensor(10.3397, device='cuda:0')) + | > loss_gen: 1.8821479082107544 (1.8775498285827794) + | > loss_kl: 2.44990873336792 (2.526801413520244) + | > loss_feat: 2.9431896209716797 (3.2870303060998562) + | > loss_mel: 21.016803741455078 (21.438969932651133) + | > loss_duration: 1.3911818265914917 (1.3741883865530564) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.6832332611084 (30.50453985380434) + | > grad_norm_1: tensor(59.3466, device='cuda:0') (tensor(118.4663, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 1.0363 (0.8992296996452996) + | > loader_time: 0.0212 (0.014130960856235867) + + + --> TIME: 2024-06-25 06:42:03 -- STEP: 507/591 -- GLOBAL_STEP: 14100 + | > loss_disc: 2.7388758659362793 (2.7411959542795516) + | > loss_disc_real_0: 0.19745126366615295 (0.2150148548318084) + | > loss_disc_real_1: 0.23211432993412018 (0.22945247023298423) + | > loss_disc_real_2: 0.23627766966819763 (0.23610533177264814) + | > loss_disc_real_3: 0.21320340037345886 (0.23603363385097045) + | > loss_disc_real_4: 0.21701322495937347 (0.23485678640107902) + | > loss_disc_real_5: 0.23358498513698578 (0.23834948388664914) + | > loss_0: 2.7388758659362793 (2.7411959542795516) + | > grad_norm_0: tensor(4.0764, device='cuda:0') (tensor(10.1320, device='cuda:0')) + | > loss_gen: 1.761646032333374 (1.8764511866447253) + | > loss_kl: 2.575213670730591 (2.5248211544646324) + | > loss_feat: 3.3303894996643066 (3.2806172841164254) + | > loss_mel: 21.109617233276367 (21.43283934659037) + | > loss_duration: 1.4344947338104248 (1.3757780921999994) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.211360931396484 (30.490507046851885) + | > grad_norm_1: tensor(43.0396, device='cuda:0') (tensor(119.0175, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 1.0393 (0.9071179024096306) + | > loader_time: 0.017 (0.014448464504595573) + + + --> TIME: 2024-06-25 06:42:31 -- STEP: 532/591 -- GLOBAL_STEP: 14125 + | > loss_disc: 2.7666022777557373 (2.742101745946067) + | > loss_disc_real_0: 0.22816047072410583 (0.21565318159422925) + | > loss_disc_real_1: 0.1970670372247696 (0.229335046576378) + | > loss_disc_real_2: 0.21969100832939148 (0.23600916345988898) + | > loss_disc_real_3: 0.17974019050598145 (0.23595500121215232) + | > loss_disc_real_4: 0.1918884962797165 (0.23480228990092317) + | > loss_disc_real_5: 0.20934364199638367 (0.23829385397353567) + | > loss_0: 2.7666022777557373 (2.742101745946067) + | > grad_norm_0: tensor(4.9433, device='cuda:0') (tensor(10.3273, device='cuda:0')) + | > loss_gen: 1.8636858463287354 (1.8765058564512354) + | > loss_kl: 2.561840295791626 (2.5221111209769007) + | > loss_feat: 3.3455893993377686 (3.2746877271429935) + | > loss_mel: 22.048410415649414 (21.42266123993953) + | > loss_duration: 1.4213489294052124 (1.377276948520116) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.240875244140625 (30.47324287084708) + | > grad_norm_1: tensor(57.7634, device='cuda:0') (tensor(118.7641, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 1.0837 (0.9164310205251652) + | > loader_time: 0.0256 (0.014782342247496869) + + + --> TIME: 2024-06-25 06:43:01 -- STEP: 557/591 -- GLOBAL_STEP: 14150 + | > loss_disc: 2.710817575454712 (2.741836347100551) + | > loss_disc_real_0: 0.11644771695137024 (0.21545030303714088) + | > loss_disc_real_1: 0.2541068196296692 (0.2294770143190448) + | > loss_disc_real_2: 0.2523995041847229 (0.2360453448882009) + | > loss_disc_real_3: 0.228916198015213 (0.2359966474118738) + | > loss_disc_real_4: 0.2507357597351074 (0.2347986030407397) + | > loss_disc_real_5: 0.2291145622730255 (0.2383759302928161) + | > loss_0: 2.710817575454712 (2.741836347100551) + | > grad_norm_0: tensor(23.6256, device='cuda:0') (tensor(10.1956, device='cuda:0')) + | > loss_gen: 1.9363563060760498 (1.8758989947060494) + | > loss_kl: 2.403937578201294 (2.5199775593071054) + | > loss_feat: 3.2553794384002686 (3.2675947334008755) + | > loss_mel: 21.654428482055664 (21.401291814697924) + | > loss_duration: 1.4036061763763428 (1.3790200931593593) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.65370750427246 (30.443783172799183) + | > grad_norm_1: tensor(120.3230, device='cuda:0') (tensor(118.3745, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 1.2477 (0.9277987878241274) + | > loader_time: 0.0234 (0.01513224572638849) + + + --> TIME: 2024-06-25 06:43:34 -- STEP: 582/591 -- GLOBAL_STEP: 14175 + | > loss_disc: 2.7558865547180176 (2.742085375327015) + | > loss_disc_real_0: 0.24502873420715332 (0.2155704353247926) + | > loss_disc_real_1: 0.2374516874551773 (0.2294574436248372) + | > loss_disc_real_2: 0.21329666674137115 (0.23600986850835204) + | > loss_disc_real_3: 0.24807186424732208 (0.2359366864222022) + | > loss_disc_real_4: 0.22385205328464508 (0.23480829768062048) + | > loss_disc_real_5: 0.23806408047676086 (0.23840486547586434) + | > loss_0: 2.7558865547180176 (2.742085375327015) + | > grad_norm_0: tensor(8.8370, device='cuda:0') (tensor(10.2338, device='cuda:0')) + | > loss_gen: 1.8390907049179077 (1.8747816378717979) + | > loss_kl: 2.430196523666382 (2.515795722040525) + | > loss_feat: 3.1241564750671387 (3.2589505181689433) + | > loss_mel: 21.274497985839844 (21.382210157990865) + | > loss_duration: 1.4044837951660156 (1.3805120095764243) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.072423934936523 (30.412250020659666) + | > grad_norm_1: tensor(39.4251, device='cuda:0') (tensor(118.2907, device='cuda:0')) + | > current_lr_0: 0.00019942578993363514 + | > current_lr_1: 0.00019942578993363514 + | > step_time: 1.3876 (0.9430679357338607) + | > loader_time: 0.0299 (0.01555670045085789) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036862961789394947 (+9.267888170607546e-05) + | > avg_loss_disc: 2.6954574635688293 (-0.06251278329402865) + | > avg_loss_disc_real_0: 0.20822405498078528 (-0.06563896733395608) + | > avg_loss_disc_real_1: 0.20983344887165314 (-0.03298542125427975) + | > avg_loss_disc_real_2: 0.22763467152067957 (-0.04147944456719338) + | > avg_loss_disc_real_3: 0.227964552793097 (-0.03936539019675969) + | > avg_loss_disc_real_4: 0.22037162235442628 (-0.05194946108980389) + | > avg_loss_disc_real_5: 0.24010697641271225 (-0.02362117551742718) + | > avg_loss_0: 2.6954574635688293 (-0.06251278329402865) + | > avg_loss_gen: 1.7960570441915633 (-0.2733905061762385) + | > avg_loss_kl: 2.5793861125377897 (+0.05370436323450001) + | > avg_loss_feat: 3.1792241969007127 (-0.009022423561583004) + | > avg_loss_mel: 21.43246679103121 (+0.27371568882718833) + | > avg_loss_duration: 1.398253676739145 (+0.005227165019258573) + | > avg_loss_1: 30.385387948218813 (+0.05023436850689933) + + + > EPOCH: 25/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 06:43:53)  + + --> TIME: 2024-06-25 06:44:07 -- STEP: 16/591 -- GLOBAL_STEP: 14200 + | > loss_disc: 2.645268201828003 (2.7332445830106735) + | > loss_disc_real_0: 0.17607346177101135 (0.2095769438892603) + | > loss_disc_real_1: 0.1973354071378708 (0.23130116891115904) + | > loss_disc_real_2: 0.21865998208522797 (0.23513829801231623) + | > loss_disc_real_3: 0.22180946171283722 (0.2362846527248621) + | > loss_disc_real_4: 0.2520494759082794 (0.23935815878212452) + | > loss_disc_real_5: 0.25655680894851685 (0.24160032253712416) + | > loss_0: 2.645268201828003 (2.7332445830106735) + | > grad_norm_0: tensor(5.6029, device='cuda:0') (tensor(6.1669, device='cuda:0')) + | > loss_gen: 1.9204310178756714 (1.902302511036396) + | > loss_kl: 2.68036150932312 (2.689626559615135) + | > loss_feat: 3.456515312194824 (3.471360430121422) + | > loss_mel: 21.50415802001953 (21.613733291625977) + | > loss_duration: 1.3911235332489014 (1.3499147072434425) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.95258903503418 (31.02693808078766) + | > grad_norm_1: tensor(60.5930, device='cuda:0') (tensor(147.4309, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.792 (0.7731446027755737) + | > loader_time: 0.0098 (0.008905231952667235) + + + --> TIME: 2024-06-25 06:44:27 -- STEP: 41/591 -- GLOBAL_STEP: 14225 + | > loss_disc: 2.775979518890381 (2.7313733682399843) + | > loss_disc_real_0: 0.10043511539697647 (0.21214244078572203) + | > loss_disc_real_1: 0.20377574861049652 (0.22922828575459922) + | > loss_disc_real_2: 0.22029320895671844 (0.23228900134563446) + | > loss_disc_real_3: 0.27354323863983154 (0.2358356154546505) + | > loss_disc_real_4: 0.21987390518188477 (0.23623010588855278) + | > loss_disc_real_5: 0.21567751467227936 (0.2393255262840085) + | > loss_0: 2.775979518890381 (2.7313733682399843) + | > grad_norm_0: tensor(35.7496, device='cuda:0') (tensor(9.3023, device='cuda:0')) + | > loss_gen: 2.09171986579895 (1.9033660946822748) + | > loss_kl: 2.5098683834075928 (2.647397047135888) + | > loss_feat: 3.481130838394165 (3.4726416832063256) + | > loss_mel: 21.502782821655273 (21.55195105948099) + | > loss_duration: 1.388748288154602 (1.352051795982733) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.97425079345703 (30.9274079392596) + | > grad_norm_1: tensor(199.1882, device='cuda:0') (tensor(148.0511, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.8054 (0.7871120383099812) + | > loader_time: 0.0101 (0.009608361779189691) + + + --> TIME: 2024-06-25 06:44:48 -- STEP: 66/591 -- GLOBAL_STEP: 14250 + | > loss_disc: 2.716458559036255 (2.729903647393891) + | > loss_disc_real_0: 0.28210723400115967 (0.21521333706649867) + | > loss_disc_real_1: 0.230624258518219 (0.22874549121567697) + | > loss_disc_real_2: 0.25237706303596497 (0.23314459531596213) + | > loss_disc_real_3: 0.2534511387348175 (0.23515246831106418) + | > loss_disc_real_4: 0.21681742370128632 (0.23490239679813385) + | > loss_disc_real_5: 0.23870661854743958 (0.2395390739495104) + | > loss_0: 2.716458559036255 (2.729903647393891) + | > grad_norm_0: tensor(13.1324, device='cuda:0') (tensor(9.7988, device='cuda:0')) + | > loss_gen: 1.821712851524353 (1.8973334630330403) + | > loss_kl: 2.491467237472534 (2.6079396009445195) + | > loss_feat: 3.5041980743408203 (3.442485307202195) + | > loss_mel: 20.929500579833984 (21.471040696808785) + | > loss_duration: 1.3604005575180054 (1.3584334127830737) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.107280731201172 (30.777232574694086) + | > grad_norm_1: tensor(220.2780, device='cuda:0') (tensor(151.9679, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.817 (0.7973954496961652) + | > loader_time: 0.0103 (0.009939728361187561) + + + --> TIME: 2024-06-25 06:45:09 -- STEP: 91/591 -- GLOBAL_STEP: 14275 + | > loss_disc: 2.656846284866333 (2.727935298458559) + | > loss_disc_real_0: 0.1599722057580948 (0.21273422478647022) + | > loss_disc_real_1: 0.2212575227022171 (0.22862305058227791) + | > loss_disc_real_2: 0.23487533628940582 (0.23385534754821233) + | > loss_disc_real_3: 0.2381182610988617 (0.23507812160712022) + | > loss_disc_real_4: 0.25790655612945557 (0.23449890724905245) + | > loss_disc_real_5: 0.2516239881515503 (0.23888616886112715) + | > loss_0: 2.656846284866333 (2.727935298458559) + | > grad_norm_0: tensor(9.4338, device='cuda:0') (tensor(8.8339, device='cuda:0')) + | > loss_gen: 1.987485647201538 (1.8920445822097443) + | > loss_kl: 2.6395785808563232 (2.580709651276305) + | > loss_feat: 3.52449369430542 (3.430929270419446) + | > loss_mel: 21.885705947875977 (21.456292833600727) + | > loss_duration: 1.4214520454406738 (1.3662304747235643) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.458715438842773 (30.72620687379942) + | > grad_norm_1: tensor(89.0896, device='cuda:0') (tensor(147.8297, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.821 (0.8047844289423345) + | > loader_time: 0.0103 (0.010167504404927351) + + + --> TIME: 2024-06-25 06:45:30 -- STEP: 116/591 -- GLOBAL_STEP: 14300 + | > loss_disc: 2.691890239715576 (2.733749767829631) + | > loss_disc_real_0: 0.18884959816932678 (0.21476353081906663) + | > loss_disc_real_1: 0.22181212902069092 (0.2287416289849528) + | > loss_disc_real_2: 0.2308698445558548 (0.2352401846698646) + | > loss_disc_real_3: 0.18512657284736633 (0.23407831068696647) + | > loss_disc_real_4: 0.25119543075561523 (0.23441718518733978) + | > loss_disc_real_5: 0.2284633219242096 (0.23887864513130025) + | > loss_0: 2.691890239715576 (2.733749767829631) + | > grad_norm_0: tensor(7.8241, device='cuda:0') (tensor(9.6799, device='cuda:0')) + | > loss_gen: 1.9270310401916504 (1.8885363967254245) + | > loss_kl: 2.5078799724578857 (2.568302041497723) + | > loss_feat: 3.424367666244507 (3.415908129050814) + | > loss_mel: 22.106054306030273 (21.449563651249335) + | > loss_duration: 1.3739488124847412 (1.368726482679104) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.33928108215332 (30.69103668475973) + | > grad_norm_1: tensor(228.7601, device='cuda:0') (tensor(144.5169, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.8367 (0.8116229772567749) + | > loader_time: 0.0109 (0.010332639875083136) + + + --> TIME: 2024-06-25 06:45:52 -- STEP: 141/591 -- GLOBAL_STEP: 14325 + | > loss_disc: 2.725022077560425 (2.737693800148388) + | > loss_disc_real_0: 0.137831449508667 (0.21455965053739276) + | > loss_disc_real_1: 0.21906693279743195 (0.22912363549496265) + | > loss_disc_real_2: 0.22479034960269928 (0.23586618414161897) + | > loss_disc_real_3: 0.24020305275917053 (0.235498679022417) + | > loss_disc_real_4: 0.23287929594516754 (0.23487313197436907) + | > loss_disc_real_5: 0.22348152101039886 (0.2390103187966854) + | > loss_0: 2.725022077560425 (2.737693800148388) + | > grad_norm_0: tensor(14.3259, device='cuda:0') (tensor(9.4991, device='cuda:0')) + | > loss_gen: 1.8137826919555664 (1.8868886604376718) + | > loss_kl: 2.581472873687744 (2.5628511313851) + | > loss_feat: 3.3171322345733643 (3.4047415763773823) + | > loss_mel: 21.04994773864746 (21.455168284422967) + | > loss_duration: 1.331589937210083 (1.3682827552159627) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.09392738342285 (30.67793242813002) + | > grad_norm_1: tensor(45.0582, device='cuda:0') (tensor(140.1354, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.851 (0.8180363178253174) + | > loader_time: 0.0111 (0.010550061016218039) + + + --> TIME: 2024-06-25 06:46:14 -- STEP: 166/591 -- GLOBAL_STEP: 14350 + | > loss_disc: 2.7528114318847656 (2.738681788904121) + | > loss_disc_real_0: 0.29163336753845215 (0.21487541564078216) + | > loss_disc_real_1: 0.2338918149471283 (0.22927214609212185) + | > loss_disc_real_2: 0.28024154901504517 (0.23659959740667458) + | > loss_disc_real_3: 0.23906366527080536 (0.23543395784245916) + | > loss_disc_real_4: 0.21334563195705414 (0.23447814924889301) + | > loss_disc_real_5: 0.24331051111221313 (0.23879497318741788) + | > loss_0: 2.7528114318847656 (2.738681788904121) + | > grad_norm_0: tensor(11.9159, device='cuda:0') (tensor(9.6327, device='cuda:0')) + | > loss_gen: 1.7802096605300903 (1.886622015970299) + | > loss_kl: 2.5376017093658447 (2.5567818607192443) + | > loss_feat: 3.1943414211273193 (3.399631662541126) + | > loss_mel: 21.251466751098633 (21.45021781002183) + | > loss_duration: 1.3708840608596802 (1.3665146705615954) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.134504318237305 (30.659768058593016) + | > grad_norm_1: tensor(83.2096, device='cuda:0') (tensor(141.3090, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.8478 (0.8239223238933517) + | > loader_time: 0.0113 (0.01071293813636504) + + + --> TIME: 2024-06-25 06:46:36 -- STEP: 191/591 -- GLOBAL_STEP: 14375 + | > loss_disc: 2.7469165325164795 (2.7384937645877208) + | > loss_disc_real_0: 0.17933157086372375 (0.21436213889203146) + | > loss_disc_real_1: 0.22822274267673492 (0.22940848120220045) + | > loss_disc_real_2: 0.2388598620891571 (0.23644837136355995) + | > loss_disc_real_3: 0.244825541973114 (0.23524755021040353) + | > loss_disc_real_4: 0.2620822787284851 (0.2346289713345273) + | > loss_disc_real_5: 0.23303675651550293 (0.2387917505196876) + | > loss_0: 2.7469165325164795 (2.7384937645877208) + | > grad_norm_0: tensor(8.7222, device='cuda:0') (tensor(9.7028, device='cuda:0')) + | > loss_gen: 1.894758701324463 (1.8871324155967273) + | > loss_kl: 2.587709665298462 (2.5500233447988627) + | > loss_feat: 3.4041247367858887 (3.3961442116043337) + | > loss_mel: 21.646982192993164 (21.427533374406917) + | > loss_duration: 1.3654385805130005 (1.3648519372440764) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.89901351928711 (30.62568531735405) + | > grad_norm_1: tensor(46.9573, device='cuda:0') (tensor(136.7727, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.8991 (0.829705923639667) + | > loader_time: 0.0142 (0.010921254832083022) + + + --> TIME: 2024-06-25 06:46:58 -- STEP: 216/591 -- GLOBAL_STEP: 14400 + | > loss_disc: 2.7872538566589355 (2.738814020598375) + | > loss_disc_real_0: 0.1891653835773468 (0.21441913337481242) + | > loss_disc_real_1: 0.2947543263435364 (0.2296882826420996) + | > loss_disc_real_2: 0.27558282017707825 (0.23653350328957592) + | > loss_disc_real_3: 0.28770560026168823 (0.23576760568000651) + | > loss_disc_real_4: 0.2615537941455841 (0.2346285311160264) + | > loss_disc_real_5: 0.2830345034599304 (0.23880183717442882) + | > loss_0: 2.7872538566589355 (2.738814020598375) + | > grad_norm_0: tensor(9.2833, device='cuda:0') (tensor(9.6495, device='cuda:0')) + | > loss_gen: 1.8927682638168335 (1.887425751045898) + | > loss_kl: 2.5260493755340576 (2.545963671472338) + | > loss_feat: 3.162590503692627 (3.3944654663403835) + | > loss_mel: 21.39571762084961 (21.434657123353755) + | > loss_duration: 1.3509583473205566 (1.364091218621643) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.32808494567871 (30.626603267810964) + | > grad_norm_1: tensor(42.9334, device='cuda:0') (tensor(133.5678, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.8695 (0.835530075762007) + | > loader_time: 0.0154 (0.01114671870514199) + + + --> TIME: 2024-06-25 06:47:21 -- STEP: 241/591 -- GLOBAL_STEP: 14425 + | > loss_disc: 2.6848511695861816 (2.7400899942485117) + | > loss_disc_real_0: 0.23135033249855042 (0.21513115880523975) + | > loss_disc_real_1: 0.21412812173366547 (0.22924298029470247) + | > loss_disc_real_2: 0.24851445853710175 (0.2363212624897106) + | > loss_disc_real_3: 0.22240442037582397 (0.23593884446808905) + | > loss_disc_real_4: 0.23316887021064758 (0.2347230877866389) + | > loss_disc_real_5: 0.24309270083904266 (0.23894552187800902) + | > loss_0: 2.6848511695861816 (2.7400899942485117) + | > grad_norm_0: tensor(6.8560, device='cuda:0') (tensor(9.9886, device='cuda:0')) + | > loss_gen: 1.7031116485595703 (1.8855134510895029) + | > loss_kl: 2.5754308700561523 (2.543527194573177) + | > loss_feat: 3.459228992462158 (3.388978256724188) + | > loss_mel: 21.779104232788086 (21.444281922336444) + | > loss_duration: 1.3673157691955566 (1.3639426523224452) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.884191513061523 (30.62624350425119) + | > grad_norm_1: tensor(118.2209, device='cuda:0') (tensor(129.9275, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.9113 (0.8418630890826466) + | > loader_time: 0.0159 (0.011459561304432726) + + + --> TIME: 2024-06-25 06:47:44 -- STEP: 266/591 -- GLOBAL_STEP: 14450 + | > loss_disc: 2.653033494949341 (2.7394093015140153) + | > loss_disc_real_0: 0.21227291226387024 (0.21484108784593137) + | > loss_disc_real_1: 0.24554762244224548 (0.2292849689273906) + | > loss_disc_real_2: 0.24821551144123077 (0.2360028944405398) + | > loss_disc_real_3: 0.23848946392536163 (0.23592745776015117) + | > loss_disc_real_4: 0.25261205434799194 (0.23466745993696658) + | > loss_disc_real_5: 0.2537260949611664 (0.2389760906422945) + | > loss_0: 2.653033494949341 (2.7394093015140153) + | > grad_norm_0: tensor(4.2795, device='cuda:0') (tensor(9.7809, device='cuda:0')) + | > loss_gen: 1.84468674659729 (1.8842156784875053) + | > loss_kl: 2.4338812828063965 (2.538203819353778) + | > loss_feat: 3.324796676635742 (3.380015199345754) + | > loss_mel: 20.873682022094727 (21.42488855347598) + | > loss_duration: 1.3602478504180908 (1.3636057995315785) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.837295532226562 (30.59092909590642) + | > grad_norm_1: tensor(68.0241, device='cuda:0') (tensor(127.6377, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.9301 (0.8477436931509721) + | > loader_time: 0.0155 (0.011763945558017361) + + + --> TIME: 2024-06-25 06:48:08 -- STEP: 291/591 -- GLOBAL_STEP: 14475 + | > loss_disc: 2.6944363117218018 (2.737958139570308) + | > loss_disc_real_0: 0.2364894449710846 (0.21456928338689088) + | > loss_disc_real_1: 0.23064427077770233 (0.22912666662452147) + | > loss_disc_real_2: 0.2258072793483734 (0.23574565504629588) + | > loss_disc_real_3: 0.2261630892753601 (0.23554216151180135) + | > loss_disc_real_4: 0.2343754768371582 (0.23466474626891806) + | > loss_disc_real_5: 0.22025670111179352 (0.23896590484581454) + | > loss_0: 2.6944363117218018 (2.737958139570308) + | > grad_norm_0: tensor(10.4831, device='cuda:0') (tensor(9.7927, device='cuda:0')) + | > loss_gen: 1.8710055351257324 (1.8838922244167) + | > loss_kl: 2.5721354484558105 (2.536119162831519) + | > loss_feat: 3.482710838317871 (3.377539732202222) + | > loss_mel: 21.317005157470703 (21.40875482723066) + | > loss_duration: 1.395850658416748 (1.3646166480283974) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.638708114624023 (30.570922622156306) + | > grad_norm_1: tensor(82.8907, device='cuda:0') (tensor(127.1059, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.9301 (0.8531316637583204) + | > loader_time: 0.0136 (0.012032369567766222) + + + --> TIME: 2024-06-25 06:48:31 -- STEP: 316/591 -- GLOBAL_STEP: 14500 + | > loss_disc: 2.7474617958068848 (2.738617804231523) + | > loss_disc_real_0: 0.19244664907455444 (0.2143863429185711) + | > loss_disc_real_1: 0.2447551190853119 (0.22929976366554636) + | > loss_disc_real_2: 0.2209828794002533 (0.23593074776515177) + | > loss_disc_real_3: 0.244054913520813 (0.23574056990350348) + | > loss_disc_real_4: 0.2663871645927429 (0.2348038093010082) + | > loss_disc_real_5: 0.25654518604278564 (0.2389688771925395) + | > loss_0: 2.7474617958068848 (2.738617804231523) + | > grad_norm_0: tensor(6.0230, device='cuda:0') (tensor(9.9026, device='cuda:0')) + | > loss_gen: 1.9392577409744263 (1.8841618820081782) + | > loss_kl: 2.536970615386963 (2.5342595079277133) + | > loss_feat: 3.2004623413085938 (3.3707401186605037) + | > loss_mel: 21.701223373413086 (21.398718761492376) + | > loss_duration: 1.387048363685608 (1.365408989070338) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.764963150024414 (30.553289250482486) + | > grad_norm_1: tensor(143.6906, device='cuda:0') (tensor(128.5436, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.9137 (0.8585488886772832) + | > loader_time: 0.017 (0.012291897701311713) + + + --> TIME: 2024-06-25 06:48:55 -- STEP: 341/591 -- GLOBAL_STEP: 14525 + | > loss_disc: 2.7443435192108154 (2.7377340779626125) + | > loss_disc_real_0: 0.17118589580059052 (0.21442489926853492) + | > loss_disc_real_1: 0.1958334594964981 (0.22926836576629595) + | > loss_disc_real_2: 0.21231797337532043 (0.23566273982224226) + | > loss_disc_real_3: 0.23013100028038025 (0.23564259048081562) + | > loss_disc_real_4: 0.25025326013565063 (0.2346755420881982) + | > loss_disc_real_5: 0.20962457358837128 (0.23874615993667558) + | > loss_0: 2.7443435192108154 (2.7377340779626125) + | > grad_norm_0: tensor(7.9172, device='cuda:0') (tensor(10.0270, device='cuda:0')) + | > loss_gen: 1.8847239017486572 (1.8840132392396676) + | > loss_kl: 2.6154372692108154 (2.5306955020099093) + | > loss_feat: 3.238664150238037 (3.3687980811267297) + | > loss_mel: 21.445261001586914 (21.404979744964322) + | > loss_duration: 1.3791511058807373 (1.36646723817171) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.563236236572266 (30.55495379048009) + | > grad_norm_1: tensor(122.0448, device='cuda:0') (tensor(128.9528, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.9373 (0.8640795173533151) + | > loader_time: 0.016 (0.012539768498664028) + + + --> TIME: 2024-06-25 06:49:19 -- STEP: 366/591 -- GLOBAL_STEP: 14550 + | > loss_disc: 2.805382490158081 (2.738274754722262) + | > loss_disc_real_0: 0.2166137844324112 (0.21460298715474832) + | > loss_disc_real_1: 0.24201983213424683 (0.2292965531267755) + | > loss_disc_real_2: 0.23431311547756195 (0.23586909082092222) + | > loss_disc_real_3: 0.26106393337249756 (0.23606855023265536) + | > loss_disc_real_4: 0.23193767666816711 (0.2346439345286844) + | > loss_disc_real_5: 0.22959131002426147 (0.23864339137337898) + | > loss_0: 2.805382490158081 (2.738274754722262) + | > grad_norm_0: tensor(3.3653, device='cuda:0') (tensor(10.0838, device='cuda:0')) + | > loss_gen: 1.8420145511627197 (1.8838009342469804) + | > loss_kl: 2.4875948429107666 (2.5295521307512705) + | > loss_feat: 3.274650812149048 (3.3621435491113707) + | > loss_mel: 21.244016647338867 (21.39546833663689) + | > loss_duration: 1.3985118865966797 (1.3681514041671352) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.246788024902344 (30.539116338302527) + | > grad_norm_1: tensor(59.9871, device='cuda:0') (tensor(128.2520, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.9396 (0.8694779846837611) + | > loader_time: 0.0151 (0.012801793755077924) + + + --> TIME: 2024-06-25 06:49:44 -- STEP: 391/591 -- GLOBAL_STEP: 14575 + | > loss_disc: 2.813304901123047 (2.7376540262071067) + | > loss_disc_real_0: 0.34686174988746643 (0.21465435124876558) + | > loss_disc_real_1: 0.2519468367099762 (0.22925653028518647) + | > loss_disc_real_2: 0.24488674104213715 (0.23591877360020758) + | > loss_disc_real_3: 0.24266645312309265 (0.23594051873897348) + | > loss_disc_real_4: 0.26787126064300537 (0.23457773986374938) + | > loss_disc_real_5: 0.27056047320365906 (0.2386460610286659) + | > loss_0: 2.813304901123047 (2.7376540262071067) + | > grad_norm_0: tensor(20.0050, device='cuda:0') (tensor(10.0632, device='cuda:0')) + | > loss_gen: 1.7646939754486084 (1.884196423203744) + | > loss_kl: 2.5191404819488525 (2.527460003143077) + | > loss_feat: 3.2524077892303467 (3.359178172962744) + | > loss_mel: 20.831783294677734 (21.38958975848029) + | > loss_duration: 1.3948917388916016 (1.3689015826300903) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.76291847229004 (30.529325943773664) + | > grad_norm_1: tensor(111.0070, device='cuda:0') (tensor(128.5947, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 0.9561 (0.8753159082759067) + | > loader_time: 0.0153 (0.013080379847065567) + + + --> TIME: 2024-06-25 06:50:09 -- STEP: 416/591 -- GLOBAL_STEP: 14600 + | > loss_disc: 2.774503469467163 (2.7387319115492015) + | > loss_disc_real_0: 0.2930411994457245 (0.21473324205726388) + | > loss_disc_real_1: 0.24541687965393066 (0.22921453616940057) + | > loss_disc_real_2: 0.2521401047706604 (0.23602855090911573) + | > loss_disc_real_3: 0.20207861065864563 (0.2359329119969446) + | > loss_disc_real_4: 0.2265504151582718 (0.23473063985315656) + | > loss_disc_real_5: 0.22679653763771057 (0.2387321679494702) + | > loss_0: 2.774503469467163 (2.7387319115492015) + | > grad_norm_0: tensor(17.8108, device='cuda:0') (tensor(10.0838, device='cuda:0')) + | > loss_gen: 1.7731349468231201 (1.8821011985150666) + | > loss_kl: 2.555767774581909 (2.525283064406653) + | > loss_feat: 3.084580421447754 (3.3478152155876155) + | > loss_mel: 20.925384521484375 (21.374033918747532) + | > loss_duration: 1.3968100547790527 (1.369886896071527) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.73567771911621 (30.499120285877815) + | > grad_norm_1: tensor(109.3247, device='cuda:0') (tensor(127.0783, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 1.0041 (0.881190090225293) + | > loader_time: 0.0176 (0.013351258750145253) + + + --> TIME: 2024-06-25 06:50:34 -- STEP: 441/591 -- GLOBAL_STEP: 14625 + | > loss_disc: 2.733809232711792 (2.7390365730337547) + | > loss_disc_real_0: 0.18462035059928894 (0.2147777129699584) + | > loss_disc_real_1: 0.21329659223556519 (0.22920531206795958) + | > loss_disc_real_2: 0.2405683994293213 (0.23596920404169294) + | > loss_disc_real_3: 0.2199184000492096 (0.23592148714460212) + | > loss_disc_real_4: 0.26552069187164307 (0.23477375267449424) + | > loss_disc_real_5: 0.2489078789949417 (0.23874665259503994) + | > loss_0: 2.733809232711792 (2.7390365730337547) + | > grad_norm_0: tensor(7.6547, device='cuda:0') (tensor(10.1789, device='cuda:0')) + | > loss_gen: 1.8365795612335205 (1.8813699378448279) + | > loss_kl: 2.5172200202941895 (2.5229340567340124) + | > loss_feat: 3.013756275177002 (3.3416493663441833) + | > loss_mel: 20.409059524536133 (21.3648690766488) + | > loss_duration: 1.386415958404541 (1.3708729284420587) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.16303253173828 (30.481695339522936) + | > grad_norm_1: tensor(119.7686, device='cuda:0') (tensor(126.2301, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 1.0011 (0.8880080549625042) + | > loader_time: 0.0151 (0.013604334963151927) + + + --> TIME: 2024-06-25 06:51:00 -- STEP: 466/591 -- GLOBAL_STEP: 14650 + | > loss_disc: 2.7674944400787354 (2.739132890373852) + | > loss_disc_real_0: 0.1533610224723816 (0.21469740024388095) + | > loss_disc_real_1: 0.24684254825115204 (0.22930181106400593) + | > loss_disc_real_2: 0.2215636521577835 (0.23585592050409113) + | > loss_disc_real_3: 0.2303493767976761 (0.23585373422234868) + | > loss_disc_real_4: 0.24211405217647552 (0.23478457421192286) + | > loss_disc_real_5: 0.2368115484714508 (0.23884289606190545) + | > loss_0: 2.7674944400787354 (2.739132890373852) + | > grad_norm_0: tensor(13.1068, device='cuda:0') (tensor(10.2297, device='cuda:0')) + | > loss_gen: 1.937356948852539 (1.881103405113384) + | > loss_kl: 2.4826247692108154 (2.519945398932364) + | > loss_feat: 3.324040174484253 (3.334677279762955) + | > loss_mel: 21.40782928466797 (21.350177302380995) + | > loss_duration: 1.4105806350708008 (1.3721986952769407) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.56243133544922 (30.45810205025734) + | > grad_norm_1: tensor(100.9480, device='cuda:0') (tensor(125.7232, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 1.0169 (0.8945879219939269) + | > loader_time: 0.0225 (0.013915602741323316) + + + --> TIME: 2024-06-25 06:51:27 -- STEP: 491/591 -- GLOBAL_STEP: 14675 + | > loss_disc: 2.7660748958587646 (2.7390444385544104) + | > loss_disc_real_0: 0.24495916068553925 (0.21488807025667123) + | > loss_disc_real_1: 0.2363349199295044 (0.2292891829113368) + | > loss_disc_real_2: 0.2496175318956375 (0.2357766775456077) + | > loss_disc_real_3: 0.2652885913848877 (0.2358659019356104) + | > loss_disc_real_4: 0.23511135578155518 (0.2347954822108605) + | > loss_disc_real_5: 0.23664554953575134 (0.23878790926787616) + | > loss_0: 2.7660748958587646 (2.7390444385544104) + | > grad_norm_0: tensor(8.1479, device='cuda:0') (tensor(10.0669, device='cuda:0')) + | > loss_gen: 1.7666969299316406 (1.8803947583721028) + | > loss_kl: 2.442047119140625 (2.518711208083237) + | > loss_feat: 3.1044795513153076 (3.3320059450969186) + | > loss_mel: 20.78207778930664 (21.343893664917484) + | > loss_duration: 1.4395637512207031 (1.3736149541229443) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.534866333007812 (30.44862050679939) + | > grad_norm_1: tensor(174.1062, device='cuda:0') (tensor(124.9242, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 1.0749 (0.9019915581720667) + | > loader_time: 0.0169 (0.014228790694972892) + + + --> TIME: 2024-06-25 06:51:55 -- STEP: 516/591 -- GLOBAL_STEP: 14700 + | > loss_disc: 2.7362921237945557 (2.740146360656088) + | > loss_disc_real_0: 0.20439501106739044 (0.2151305253451416) + | > loss_disc_real_1: 0.21582292020320892 (0.22927715638002685) + | > loss_disc_real_2: 0.2046757936477661 (0.23571246538975443) + | > loss_disc_real_3: 0.17946167290210724 (0.23582738461767055) + | > loss_disc_real_4: 0.23682640492916107 (0.2348844017111516) + | > loss_disc_real_5: 0.23077335953712463 (0.23894693272173867) + | > loss_0: 2.7362921237945557 (2.740146360656088) + | > grad_norm_0: tensor(4.6487, device='cuda:0') (tensor(10.2046, device='cuda:0')) + | > loss_gen: 1.8629848957061768 (1.8798469161802485) + | > loss_kl: 2.4792356491088867 (2.516603418560916) + | > loss_feat: 3.39505672454834 (3.327441362909568) + | > loss_mel: 21.412464141845703 (21.34401889180028) + | > loss_duration: 1.397001028060913 (1.3752478259478438) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.546743392944336 (30.44315839737885) + | > grad_norm_1: tensor(96.7510, device='cuda:0') (tensor(124.4180, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 1.0788 (0.9122768058333286) + | > loader_time: 0.0184 (0.01451412651890008) + + + --> TIME: 2024-06-25 06:52:25 -- STEP: 541/591 -- GLOBAL_STEP: 14725 + | > loss_disc: 2.79221510887146 (2.7411741716804428) + | > loss_disc_real_0: 0.27229833602905273 (0.21535156726451551) + | > loss_disc_real_1: 0.2975955307483673 (0.22943766982520133) + | > loss_disc_real_2: 0.23418621718883514 (0.23586818719086497) + | > loss_disc_real_3: 0.25221601128578186 (0.23595454448803074) + | > loss_disc_real_4: 0.268160879611969 (0.23501419348989974) + | > loss_disc_real_5: 0.24262826144695282 (0.23896508786462367) + | > loss_0: 2.79221510887146 (2.7411741716804428) + | > grad_norm_0: tensor(9.7839, device='cuda:0') (tensor(10.2312, device='cuda:0')) + | > loss_gen: 1.721737027168274 (1.8784326070337773) + | > loss_kl: 2.484968662261963 (2.5148438966648867) + | > loss_feat: 2.979506254196167 (3.3170637299966015) + | > loss_mel: 20.56672477722168 (21.328095709330047) + | > loss_duration: 1.395774006843567 (1.3767672473093062) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.148712158203125 (30.415203189673573) + | > grad_norm_1: tensor(103.3137, device='cuda:0') (tensor(125.4582, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 1.1538 (0.9238418946645175) + | > loader_time: 0.0207 (0.014852772358384898) + + + --> TIME: 2024-06-25 06:52:56 -- STEP: 566/591 -- GLOBAL_STEP: 14750 + | > loss_disc: 2.785876750946045 (2.74235961925857) + | > loss_disc_real_0: 0.2918056547641754 (0.21539719720032113) + | > loss_disc_real_1: 0.22910122573375702 (0.2293952111378576) + | > loss_disc_real_2: 0.21353627741336823 (0.23609589735315883) + | > loss_disc_real_3: 0.2416507601737976 (0.23609212498993418) + | > loss_disc_real_4: 0.19959278404712677 (0.23497798146396986) + | > loss_disc_real_5: 0.22000305354595184 (0.2390277383342227) + | > loss_0: 2.785876750946045 (2.74235961925857) + | > grad_norm_0: tensor(12.1307, device='cuda:0') (tensor(10.1986, device='cuda:0')) + | > loss_gen: 1.7569993734359741 (1.8770465132625702) + | > loss_kl: 2.318268060684204 (2.5122751540935524) + | > loss_feat: 2.99464750289917 (3.305363321894049) + | > loss_mel: 20.844974517822266 (21.309896930788934) + | > loss_duration: 1.4203927516937256 (1.3783800660932028) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.335283279418945 (30.382962004455997) + | > grad_norm_1: tensor(190.7749, device='cuda:0') (tensor(126.3845, device='cuda:0')) + | > current_lr_0: 0.00019940086170989343 + | > current_lr_1: 0.00019940086170989343 + | > step_time: 1.2698 (0.9366490095327259) + | > loader_time: 0.0253 (0.015222768480280684) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003684962049443671 (-1.334129495823757e-06) + | > avg_loss_disc: 2.715340533155076 (+0.019883069586246638) + | > avg_loss_disc_real_0: 0.2729297416641358 (+0.06470568668335053) + | > avg_loss_disc_real_1: 0.20779891660872923 (-0.0020345322629239104) + | > avg_loss_disc_real_2: 0.20854534208774567 (-0.0190893294329339) + | > avg_loss_disc_real_3: 0.21490349065750203 (-0.013061062135594964) + | > avg_loss_disc_real_4: 0.22458127109294243 (+0.004209648738516147) + | > avg_loss_disc_real_5: 0.23556398552782992 (-0.004542990884882331) + | > avg_loss_0: 2.715340533155076 (+0.019883069586246638) + | > avg_loss_gen: 1.8671675641485985 (+0.07111051995703521) + | > avg_loss_kl: 2.593032177458418 (+0.013646064920628476) + | > avg_loss_feat: 3.3067615184378116 (+0.12753732153709896) + | > avg_loss_mel: 21.303366437871396 (-0.1291003531598136) + | > avg_loss_duration: 1.3739297491438844 (-0.024323927595260475) + | > avg_loss_1: 30.44425749271474 (+0.05886954449592707) + + + > EPOCH: 26/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 06:53:36)  + + --> TIME: 2024-06-25 06:53:37 -- STEP: 0/591 -- GLOBAL_STEP: 14775 + | > loss_disc: 2.7110188007354736 (2.7110188007354736) + | > loss_disc_real_0: 0.2665773928165436 (0.2665773928165436) + | > loss_disc_real_1: 0.18129493296146393 (0.18129493296146393) + | > loss_disc_real_2: 0.18976840376853943 (0.18976840376853943) + | > loss_disc_real_3: 0.20205791294574738 (0.20205791294574738) + | > loss_disc_real_4: 0.22741983830928802 (0.22741983830928802) + | > loss_disc_real_5: 0.2189611941576004 (0.2189611941576004) + | > loss_0: 2.7110188007354736 (2.7110188007354736) + | > grad_norm_0: tensor(12.8077, device='cuda:0') (tensor(12.8077, device='cuda:0')) + | > loss_gen: 2.1534032821655273 (2.1534032821655273) + | > loss_kl: 2.808586359024048 (2.808586359024048) + | > loss_feat: 3.542227268218994 (3.542227268218994) + | > loss_mel: 21.667789459228516 (21.667789459228516) + | > loss_duration: 1.3519537448883057 (1.3519537448883057) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.523962020874023 (31.523962020874023) + | > grad_norm_1: tensor(151.0266, device='cuda:0') (tensor(151.0266, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.8435 (0.8435359001159668) + | > loader_time: 0.6407 (0.6407103538513184) + + + --> TIME: 2024-06-25 06:53:57 -- STEP: 25/591 -- GLOBAL_STEP: 14800 + | > loss_disc: 2.705108880996704 (2.7483314418792717) + | > loss_disc_real_0: 0.18383212387561798 (0.21401101231575012) + | > loss_disc_real_1: 0.22062493860721588 (0.23703718423843384) + | > loss_disc_real_2: 0.23100434243679047 (0.23338613629341126) + | > loss_disc_real_3: 0.22262147068977356 (0.23543552160263062) + | > loss_disc_real_4: 0.20927511155605316 (0.2371739035844803) + | > loss_disc_real_5: 0.21617043018341064 (0.24006847143173218) + | > loss_0: 2.705108880996704 (2.7483314418792717) + | > grad_norm_0: tensor(6.8611, device='cuda:0') (tensor(10.6244, device='cuda:0')) + | > loss_gen: 1.930410385131836 (1.8911215353012085) + | > loss_kl: 2.6006016731262207 (2.6499246406555175) + | > loss_feat: 3.532762050628662 (3.4688962650299073) + | > loss_mel: 20.668699264526367 (21.44513214111328) + | > loss_duration: 1.3010132312774658 (1.33884831905365) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.03348731994629 (30.793923110961913) + | > grad_norm_1: tensor(117.8094, device='cuda:0') (tensor(83.0135, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.797 (0.7894619560241699) + | > loader_time: 0.0095 (0.008739089965820313) + + + --> TIME: 2024-06-25 06:54:18 -- STEP: 50/591 -- GLOBAL_STEP: 14825 + | > loss_disc: 2.700554847717285 (2.733364486694336) + | > loss_disc_real_0: 0.18730412423610687 (0.21371242612600327) + | > loss_disc_real_1: 0.21900664269924164 (0.23126021444797515) + | > loss_disc_real_2: 0.2021547108888626 (0.23310300439596177) + | > loss_disc_real_3: 0.1623326689004898 (0.2352563214302063) + | > loss_disc_real_4: 0.26117026805877686 (0.2360367703437805) + | > loss_disc_real_5: 0.25343969464302063 (0.23960409045219422) + | > loss_0: 2.700554847717285 (2.733364486694336) + | > grad_norm_0: tensor(7.0764, device='cuda:0') (tensor(9.9782, device='cuda:0')) + | > loss_gen: 1.9235624074935913 (1.8941869831085205) + | > loss_kl: 2.643895387649536 (2.6074239778518677) + | > loss_feat: 3.5258288383483887 (3.455288290977478) + | > loss_mel: 21.334972381591797 (21.4502632522583) + | > loss_duration: 1.3914591073989868 (1.351675157546997) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.819717407226562 (30.758837852478027) + | > grad_norm_1: tensor(237.9171, device='cuda:0') (tensor(97.6473, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.8104 (0.7985927295684815) + | > loader_time: 0.011 (0.009323859214782713) + + + --> TIME: 2024-06-25 06:54:39 -- STEP: 75/591 -- GLOBAL_STEP: 14850 + | > loss_disc: 2.778852939605713 (2.736645517349243) + | > loss_disc_real_0: 0.21602608263492584 (0.21528160293896992) + | > loss_disc_real_1: 0.23844248056411743 (0.23144815305868785) + | > loss_disc_real_2: 0.28518471121788025 (0.2339677502711614) + | > loss_disc_real_3: 0.26986098289489746 (0.23607497255007426) + | > loss_disc_real_4: 0.2532051205635071 (0.23466176847616832) + | > loss_disc_real_5: 0.24865953624248505 (0.23873613715171815) + | > loss_0: 2.778852939605713 (2.736645517349243) + | > grad_norm_0: tensor(4.6375, device='cuda:0') (tensor(10.2943, device='cuda:0')) + | > loss_gen: 1.7672127485275269 (1.8943897326787313) + | > loss_kl: 2.6848466396331787 (2.588884391784668) + | > loss_feat: 3.207202672958374 (3.4417797565460204) + | > loss_mel: 21.474018096923828 (21.439207127889) + | > loss_duration: 1.3831870555877686 (1.3558967733383178) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.51646614074707 (30.720157877604166) + | > grad_norm_1: tensor(102.2217, device='cuda:0') (tensor(105.6944, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.8178 (0.8048140557607015) + | > loader_time: 0.0115 (0.00970330556233724) + + + --> TIME: 2024-06-25 06:55:00 -- STEP: 100/591 -- GLOBAL_STEP: 14875 + | > loss_disc: 2.7879276275634766 (2.736030945777893) + | > loss_disc_real_0: 0.2896507978439331 (0.21610966756939887) + | > loss_disc_real_1: 0.24440927803516388 (0.23002981632947922) + | > loss_disc_real_2: 0.2562481462955475 (0.23376247823238372) + | > loss_disc_real_3: 0.27406060695648193 (0.2359583519399166) + | > loss_disc_real_4: 0.19594214856624603 (0.23420535191893577) + | > loss_disc_real_5: 0.2562359869480133 (0.23867302983999253) + | > loss_0: 2.7879276275634766 (2.736030945777893) + | > grad_norm_0: tensor(12.8259, device='cuda:0') (tensor(10.7207, device='cuda:0')) + | > loss_gen: 1.749780297279358 (1.8917271065711976) + | > loss_kl: 2.4847795963287354 (2.5682282662391662) + | > loss_feat: 3.0154082775115967 (3.441585512161255) + | > loss_mel: 20.263286590576172 (21.424298973083495) + | > loss_duration: 1.3537390232086182 (1.3637497723102576) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 28.866992950439453 (30.689589672088623) + | > grad_norm_1: tensor(167.7619, device='cuda:0') (tensor(110.0471, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.8373 (0.8115068054199219) + | > loader_time: 0.0107 (0.01001335859298706) + + + --> TIME: 2024-06-25 06:55:21 -- STEP: 125/591 -- GLOBAL_STEP: 14900 + | > loss_disc: 2.681626558303833 (2.7349062538146973) + | > loss_disc_real_0: 0.30032220482826233 (0.2162975127696991) + | > loss_disc_real_1: 0.21957072615623474 (0.22948999750614166) + | > loss_disc_real_2: 0.22844962775707245 (0.23328682792186736) + | > loss_disc_real_3: 0.2223440408706665 (0.2352697002887726) + | > loss_disc_real_4: 0.24448996782302856 (0.23433691608905793) + | > loss_disc_real_5: 0.2480027973651886 (0.23869884181022644) + | > loss_0: 2.681626558303833 (2.7349062538146973) + | > grad_norm_0: tensor(14.9398, device='cuda:0') (tensor(10.2194, device='cuda:0')) + | > loss_gen: 1.8159689903259277 (1.8888934450149537) + | > loss_kl: 2.5334835052490234 (2.5534755210876465) + | > loss_feat: 3.5381040573120117 (3.4314952964782717) + | > loss_mel: 21.420888900756836 (21.431115447998046) + | > loss_duration: 1.4184834957122803 (1.3660368204116826) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.726930618286133 (30.671016571044923) + | > grad_norm_1: tensor(220.4462, device='cuda:0') (tensor(109.1336, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.8632 (0.8180240421295166) + | > loader_time: 0.0128 (0.010268707275390625) + + + --> TIME: 2024-06-25 06:55:43 -- STEP: 150/591 -- GLOBAL_STEP: 14925 + | > loss_disc: 2.7375147342681885 (2.734880493481953) + | > loss_disc_real_0: 0.2269718050956726 (0.21624009291330973) + | > loss_disc_real_1: 0.18662407994270325 (0.22875826746225358) + | > loss_disc_real_2: 0.22064079344272614 (0.23339201718568803) + | > loss_disc_real_3: 0.2567925751209259 (0.23562190413475037) + | > loss_disc_real_4: 0.22827385365962982 (0.23400919487078983) + | > loss_disc_real_5: 0.24714940786361694 (0.23886129448811214) + | > loss_0: 2.7375147342681885 (2.734880493481953) + | > grad_norm_0: tensor(4.2155, device='cuda:0') (tensor(10.1891, device='cuda:0')) + | > loss_gen: 1.9590513706207275 (1.8890134700139363) + | > loss_kl: 2.576549768447876 (2.545755429267884) + | > loss_feat: 3.610074043273926 (3.4316272290547687) + | > loss_mel: 21.826507568359375 (21.4160577138265) + | > loss_duration: 1.3133680820465088 (1.3647237110137944) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.28554916381836 (30.64717751820882) + | > grad_norm_1: tensor(117.3907, device='cuda:0') (tensor(111.2164, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.8629 (0.8237344566980997) + | > loader_time: 0.011 (0.010533951123555502) + + + --> TIME: 2024-06-25 06:56:05 -- STEP: 175/591 -- GLOBAL_STEP: 14950 + | > loss_disc: 2.7613821029663086 (2.736781230654035) + | > loss_disc_real_0: 0.23759329319000244 (0.21631804840905325) + | > loss_disc_real_1: 0.2298520803451538 (0.22915135996682304) + | > loss_disc_real_2: 0.2405136376619339 (0.23420800600733077) + | > loss_disc_real_3: 0.25734639167785645 (0.2358729349715369) + | > loss_disc_real_4: 0.2166583240032196 (0.23387335768767767) + | > loss_disc_real_5: 0.22205910086631775 (0.2387313505581447) + | > loss_0: 2.7613821029663086 (2.736781230654035) + | > grad_norm_0: tensor(6.6181, device='cuda:0') (tensor(10.1480, device='cuda:0')) + | > loss_gen: 1.7016949653625488 (1.8858897938047137) + | > loss_kl: 2.647230863571167 (2.544079050336566) + | > loss_feat: 3.2701852321624756 (3.4211224542345318) + | > loss_mel: 21.05315399169922 (21.415006942749017) + | > loss_duration: 1.3520634174346924 (1.3631578663417272) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.024328231811523 (30.629256134033202) + | > grad_norm_1: tensor(160.7181, device='cuda:0') (tensor(114.4123, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.8778 (0.8291606875828335) + | > loader_time: 0.0128 (0.010733902794974194) + + + --> TIME: 2024-06-25 06:56:27 -- STEP: 200/591 -- GLOBAL_STEP: 14975 + | > loss_disc: 2.738539457321167 (2.737533539533615) + | > loss_disc_real_0: 0.20414474606513977 (0.21635304749011994) + | > loss_disc_real_1: 0.22276119887828827 (0.22943120419979096) + | > loss_disc_real_2: 0.20417438447475433 (0.23431173674762248) + | > loss_disc_real_3: 0.27079275250434875 (0.23596665419638158) + | > loss_disc_real_4: 0.27068856358528137 (0.2338864306360483) + | > loss_disc_real_5: 0.2726549506187439 (0.23888933002948762) + | > loss_0: 2.738539457321167 (2.737533539533615) + | > grad_norm_0: tensor(4.4519, device='cuda:0') (tensor(10.2399, device='cuda:0')) + | > loss_gen: 1.8194270133972168 (1.8885846221446991) + | > loss_kl: 2.4707815647125244 (2.543737004995347) + | > loss_feat: 3.328087329864502 (3.419917720556259) + | > loss_mel: 21.53282356262207 (21.431730728149414) + | > loss_duration: 1.342664361000061 (1.362796950340271) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.493783950805664 (30.646767024993895) + | > grad_norm_1: tensor(210.4557, device='cuda:0') (tensor(119.3451, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.8829 (0.8344863653182983) + | > loader_time: 0.0154 (0.011003580093383793) + + + --> TIME: 2024-06-25 06:56:50 -- STEP: 225/591 -- GLOBAL_STEP: 15000 + | > loss_disc: 2.662475347518921 (2.7358317926194933) + | > loss_disc_real_0: 0.15490272641181946 (0.21591066774394777) + | > loss_disc_real_1: 0.23277288675308228 (0.22898671408494314) + | > loss_disc_real_2: 0.22330528497695923 (0.23484821339448292) + | > loss_disc_real_3: 0.22511713206768036 (0.23558513647980162) + | > loss_disc_real_4: 0.20598441362380981 (0.23359426803059047) + | > loss_disc_real_5: 0.18825100362300873 (0.2385404716597663) + | > loss_0: 2.662475347518921 (2.7358317926194933) + | > grad_norm_0: tensor(11.2159, device='cuda:0') (tensor(10.3424, device='cuda:0')) + | > loss_gen: 2.0078210830688477 (1.8890879185994467) + | > loss_kl: 2.4110140800476074 (2.5379636054568824) + | > loss_feat: 3.588834047317505 (3.414751453399658) + | > loss_mel: 21.654850006103516 (21.42769024319119) + | > loss_duration: 1.3619260787963867 (1.3621747567918565) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.024444580078125 (30.63166802300347) + | > grad_norm_1: tensor(126.2414, device='cuda:0') (tensor(122.0831, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.8729 (0.8395692443847657) + | > loader_time: 0.0147 (0.011351443396674266) + + + > CHECKPOINT : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/checkpoint_15000.pth + + --> TIME: 2024-06-25 06:57:15 -- STEP: 250/591 -- GLOBAL_STEP: 15025 + | > loss_disc: 2.7138779163360596 (2.734648199081421) + | > loss_disc_real_0: 0.16739881038665771 (0.21539802750945092) + | > loss_disc_real_1: 0.21867363154888153 (0.22864294552803038) + | > loss_disc_real_2: 0.2012755572795868 (0.23496773928403855) + | > loss_disc_real_3: 0.20580849051475525 (0.23565364784002304) + | > loss_disc_real_4: 0.2277406007051468 (0.2336944162249565) + | > loss_disc_real_5: 0.2222174108028412 (0.23878303617239) + | > loss_0: 2.7138779163360596 (2.734648199081421) + | > grad_norm_0: tensor(10.9692, device='cuda:0') (tensor(10.0301, device='cuda:0')) + | > loss_gen: 2.006269931793213 (1.8899930729866028) + | > loss_kl: 2.581136465072632 (2.5370282268524176) + | > loss_feat: 3.453234910964966 (3.41879182434082) + | > loss_mel: 21.382587432861328 (21.418876197814942) + | > loss_duration: 1.3317995071411133 (1.3626339435577393) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.755027770996094 (30.627323272705077) + | > grad_norm_1: tensor(213.5593, device='cuda:0') (tensor(121.1322, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.8864 (0.8447549486160278) + | > loader_time: 0.0156 (0.011642154693603518) + + + --> TIME: 2024-06-25 06:57:38 -- STEP: 275/591 -- GLOBAL_STEP: 15050 + | > loss_disc: 2.735616445541382 (2.7374053131450307) + | > loss_disc_real_0: 0.25012004375457764 (0.2160375649007884) + | > loss_disc_real_1: 0.21341292560100555 (0.2292391762408343) + | > loss_disc_real_2: 0.2374037355184555 (0.2354614409533414) + | > loss_disc_real_3: 0.2707379460334778 (0.23572614068334752) + | > loss_disc_real_4: 0.25438612699508667 (0.23386551195924937) + | > loss_disc_real_5: 0.24478216469287872 (0.23879401299086483) + | > loss_0: 2.735616445541382 (2.7374053131450307) + | > grad_norm_0: tensor(6.3765, device='cuda:0') (tensor(10.1625, device='cuda:0')) + | > loss_gen: 1.7155179977416992 (1.8880659502202815) + | > loss_kl: 2.4798717498779297 (2.5346230532906278) + | > loss_feat: 3.099526882171631 (3.406309670535001) + | > loss_mel: 20.544614791870117 (21.402250504927203) + | > loss_duration: 1.3682987689971924 (1.3627982889522205) + | > amp_scaler: 1024.0 (532.4800000000001) + | > loss_1: 29.20783042907715 (30.594047477028585) + | > grad_norm_1: tensor(64.3289, device='cuda:0') (tensor(120.0634, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.9144 (0.8495658241618763) + | > loader_time: 0.0153 (0.011952969811179427) + + + --> TIME: 2024-06-25 06:58:02 -- STEP: 300/591 -- GLOBAL_STEP: 15075 + | > loss_disc: 2.717486619949341 (2.7375776847203563) + | > loss_disc_real_0: 0.17398209869861603 (0.21580944038927555) + | > loss_disc_real_1: 0.24351300299167633 (0.22928858066598573) + | > loss_disc_real_2: 0.246918186545372 (0.2354438756406307) + | > loss_disc_real_3: 0.2292451113462448 (0.2355580885708332) + | > loss_disc_real_4: 0.24219152331352234 (0.23384770308931674) + | > loss_disc_real_5: 0.23991192877292633 (0.23871058404445647) + | > loss_0: 2.717486619949341 (2.7375776847203563) + | > grad_norm_0: tensor(11.9171, device='cuda:0') (tensor(10.1646, device='cuda:0')) + | > loss_gen: 1.8937865495681763 (1.8869618022441863) + | > loss_kl: 2.5135695934295654 (2.532699144681295) + | > loss_feat: 3.232846736907959 (3.402276542186738) + | > loss_mel: 20.734712600708008 (21.37462793986003) + | > loss_duration: 1.3788626194000244 (1.3630542945861814) + | > amp_scaler: 1024.0 (573.44) + | > loss_1: 29.7537784576416 (30.55961976369222) + | > grad_norm_1: tensor(173.1524, device='cuda:0') (tensor(118.8329, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.9124 (0.8549027641614282) + | > loader_time: 0.0144 (0.012233924865722659) + + + --> TIME: 2024-06-25 06:58:25 -- STEP: 325/591 -- GLOBAL_STEP: 15100 + | > loss_disc: 2.7177677154541016 (2.737259730559128) + | > loss_disc_real_0: 0.24708564579486847 (0.21580246602113431) + | > loss_disc_real_1: 0.21821030974388123 (0.22926147625996515) + | > loss_disc_real_2: 0.23832400143146515 (0.235524752506843) + | > loss_disc_real_3: 0.26462671160697937 (0.23563267359366782) + | > loss_disc_real_4: 0.21987169981002808 (0.23372927927053896) + | > loss_disc_real_5: 0.24213431775569916 (0.2386902360732739) + | > loss_0: 2.7177677154541016 (2.737259730559128) + | > grad_norm_0: tensor(8.2944, device='cuda:0') (tensor(10.1315, device='cuda:0')) + | > loss_gen: 1.8387783765792847 (1.886646761894226) + | > loss_kl: 2.3805317878723145 (2.5303252007411086) + | > loss_feat: 3.4167768955230713 (3.3978645552121676) + | > loss_mel: 21.047683715820312 (21.379003730187055) + | > loss_duration: 1.4066672325134277 (1.3637409184529226) + | > amp_scaler: 512.0 (587.6184615384617) + | > loss_1: 30.090438842773438 (30.557581241314228) + | > grad_norm_1: tensor(44.1323, device='cuda:0') (tensor(116.7697, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.9363 (0.859666401789739) + | > loader_time: 0.0149 (0.012470238758967479) + + + --> TIME: 2024-06-25 06:58:49 -- STEP: 350/591 -- GLOBAL_STEP: 15125 + | > loss_disc: 2.8193359375 (2.7398247677939267) + | > loss_disc_real_0: 0.17496976256370544 (0.21693901010922023) + | > loss_disc_real_1: 0.24724642932415009 (0.22931205349309103) + | > loss_disc_real_2: 0.22333039343357086 (0.23540150157042913) + | > loss_disc_real_3: 0.14955578744411469 (0.23562086139406477) + | > loss_disc_real_4: 0.25274890661239624 (0.23365582044635505) + | > loss_disc_real_5: 0.23438288271427155 (0.23861832184450968) + | > loss_0: 2.8193359375 (2.7398247677939267) + | > grad_norm_0: tensor(8.5705, device='cuda:0') (tensor(10.4760, device='cuda:0')) + | > loss_gen: 1.8060981035232544 (1.8855087324551174) + | > loss_kl: 2.4669833183288574 (2.5283208377020716) + | > loss_feat: 3.0064525604248047 (3.389547414779663) + | > loss_mel: 20.818078994750977 (21.360869859967927) + | > loss_duration: 1.401350498199463 (1.365294130529676) + | > amp_scaler: 512.0 (582.2171428571429) + | > loss_1: 29.498964309692383 (30.529541053771972) + | > grad_norm_1: tensor(160.0991, device='cuda:0') (tensor(117.3116, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.943 (0.8649512488501416) + | > loader_time: 0.015 (0.012710187094552182) + + + --> TIME: 2024-06-25 06:59:13 -- STEP: 375/591 -- GLOBAL_STEP: 15150 + | > loss_disc: 2.655581474304199 (2.739469581604002) + | > loss_disc_real_0: 0.21358612179756165 (0.21674827579657235) + | > loss_disc_real_1: 0.24037392437458038 (0.2292028625011444) + | > loss_disc_real_2: 0.26202502846717834 (0.23524288630485535) + | > loss_disc_real_3: 0.23353616893291473 (0.23585564470291137) + | > loss_disc_real_4: 0.219896599650383 (0.23389990099271143) + | > loss_disc_real_5: 0.2167535126209259 (0.23866667493184407) + | > loss_0: 2.655581474304199 (2.739469581604002) + | > grad_norm_0: tensor(3.8582, device='cuda:0') (tensor(10.0923, device='cuda:0')) + | > loss_gen: 1.9395698308944702 (1.883676514307658) + | > loss_kl: 2.5822203159332275 (2.524526236852011) + | > loss_feat: 3.29447340965271 (3.3795520725250245) + | > loss_mel: 21.230438232421875 (21.3463134358724) + | > loss_duration: 1.3824671506881714 (1.3665651127497356) + | > amp_scaler: 512.0 (577.5360000000004) + | > loss_1: 30.429168701171875 (30.500633438110352) + | > grad_norm_1: tensor(157.7578, device='cuda:0') (tensor(117.9290, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.9532 (0.8703933842976891) + | > loader_time: 0.018 (0.013005711237589522) + + + --> TIME: 2024-06-25 06:59:38 -- STEP: 400/591 -- GLOBAL_STEP: 15175 + | > loss_disc: 2.720283269882202 (2.7385105931758864) + | > loss_disc_real_0: 0.22075192630290985 (0.21641563218086957) + | > loss_disc_real_1: 0.24046790599822998 (0.2291441524401307) + | > loss_disc_real_2: 0.23624271154403687 (0.2353226662054658) + | > loss_disc_real_3: 0.2375594675540924 (0.23585941322147846) + | > loss_disc_real_4: 0.2403554916381836 (0.23391179852187638) + | > loss_disc_real_5: 0.27635160088539124 (0.23861411415040493) + | > loss_0: 2.720283269882202 (2.7385105931758864) + | > grad_norm_0: tensor(3.5006, device='cuda:0') (tensor(9.9129, device='cuda:0')) + | > loss_gen: 1.852806806564331 (1.8837567135691642) + | > loss_kl: 2.5767853260040283 (2.5231984549760837) + | > loss_feat: 3.295194387435913 (3.375857155919075) + | > loss_mel: 21.40118408203125 (21.33780140876771) + | > loss_duration: 1.3971047401428223 (1.3676368173956872) + | > amp_scaler: 512.0 (573.4400000000004) + | > loss_1: 30.523073196411133 (30.488250613212585) + | > grad_norm_1: tensor(94.4426, device='cuda:0') (tensor(116.3936, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 0.9474 (0.876366204619408) + | > loader_time: 0.0183 (0.013282811045646672) + + + --> TIME: 2024-06-25 07:00:04 -- STEP: 425/591 -- GLOBAL_STEP: 15200 + | > loss_disc: 2.7557551860809326 (2.7388988618289707) + | > loss_disc_real_0: 0.2175929844379425 (0.21616591535946902) + | > loss_disc_real_1: 0.28005096316337585 (0.22938346245709587) + | > loss_disc_real_2: 0.23407994210720062 (0.23540884273893692) + | > loss_disc_real_3: 0.22832204401493073 (0.23603629974757923) + | > loss_disc_real_4: 0.21362309157848358 (0.2338623981966692) + | > loss_disc_real_5: 0.23369750380516052 (0.23848579171825857) + | > loss_0: 2.7557551860809326 (2.7388988618289707) + | > grad_norm_0: tensor(3.6206, device='cuda:0') (tensor(9.8899, device='cuda:0')) + | > loss_gen: 1.8166892528533936 (1.8835579437368057) + | > loss_kl: 2.385852098464966 (2.5215860529506924) + | > loss_feat: 3.2073514461517334 (3.3694337878507725) + | > loss_mel: 21.21177101135254 (21.32290459127988) + | > loss_duration: 1.3922605514526367 (1.3684236759297987) + | > amp_scaler: 512.0 (569.8258823529412) + | > loss_1: 30.01392364501953 (30.46590613421272) + | > grad_norm_1: tensor(224.5490, device='cuda:0') (tensor(118.6915, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 1.0148 (0.8838939122592704) + | > loader_time: 0.016 (0.013541983997120581) + + + --> TIME: 2024-06-25 07:00:30 -- STEP: 450/591 -- GLOBAL_STEP: 15225 + | > loss_disc: 2.7845511436462402 (2.7384871790144163) + | > loss_disc_real_0: 0.2593420147895813 (0.21618092581629753) + | > loss_disc_real_1: 0.2093478888273239 (0.2290847235586908) + | > loss_disc_real_2: 0.27732613682746887 (0.23550746589899063) + | > loss_disc_real_3: 0.22452817857265472 (0.23601503448353872) + | > loss_disc_real_4: 0.2576339542865753 (0.2339300448364682) + | > loss_disc_real_5: 0.24112024903297424 (0.23855915364291932) + | > loss_0: 2.7845511436462402 (2.7384871790144163) + | > grad_norm_0: tensor(6.4615, device='cuda:0') (tensor(10.0033, device='cuda:0')) + | > loss_gen: 1.7688877582550049 (1.8840163938204448) + | > loss_kl: 2.383944034576416 (2.5165059137344374) + | > loss_feat: 2.9824154376983643 (3.363682804637485) + | > loss_mel: 20.691404342651367 (21.308422970241978) + | > loss_duration: 1.3852189779281616 (1.3695546605851916) + | > amp_scaler: 512.0 (566.6133333333336) + | > loss_1: 29.211872100830078 (30.44218281639947) + | > grad_norm_1: tensor(97.4919, device='cuda:0') (tensor(118.6062, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 1.0086 (0.8915410608715484) + | > loader_time: 0.0226 (0.01382545047336155) + + + --> TIME: 2024-06-25 07:00:56 -- STEP: 475/591 -- GLOBAL_STEP: 15250 + | > loss_disc: 2.716317653656006 (2.7385593283803824) + | > loss_disc_real_0: 0.25710731744766235 (0.21593146220633858) + | > loss_disc_real_1: 0.20915944874286652 (0.22916186884829873) + | > loss_disc_real_2: 0.23916935920715332 (0.2355637427693919) + | > loss_disc_real_3: 0.2274160087108612 (0.23584880938655453) + | > loss_disc_real_4: 0.23029279708862305 (0.23395444600205675) + | > loss_disc_real_5: 0.2383660227060318 (0.23851102471351623) + | > loss_0: 2.716317653656006 (2.7385593283803824) + | > grad_norm_0: tensor(11.6588, device='cuda:0') (tensor(10.0203, device='cuda:0')) + | > loss_gen: 1.9558982849121094 (1.8832036485170063) + | > loss_kl: 2.3034045696258545 (2.5135745088677672) + | > loss_feat: 3.3935720920562744 (3.35644749842192) + | > loss_mel: 21.537643432617188 (21.293881036858817) + | > loss_duration: 1.4161274433135986 (1.370789337660137) + | > amp_scaler: 512.0 (563.7389473684216) + | > loss_1: 30.606645584106445 (30.417896098086707) + | > grad_norm_1: tensor(181.5911, device='cuda:0') (tensor(120.5275, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 1.0011 (0.8984736186579656) + | > loader_time: 0.0179 (0.014098249234651267) + + + --> TIME: 2024-06-25 07:01:23 -- STEP: 500/591 -- GLOBAL_STEP: 15275 + | > loss_disc: 2.7048745155334473 (2.738478866577147) + | > loss_disc_real_0: 0.2285253256559372 (0.2157826964557171) + | > loss_disc_real_1: 0.22393636405467987 (0.22923556914925575) + | > loss_disc_real_2: 0.2540614902973175 (0.23569543653726577) + | > loss_disc_real_3: 0.21823230385780334 (0.2359378420114517) + | > loss_disc_real_4: 0.24107472598552704 (0.23390718626976015) + | > loss_disc_real_5: 0.2435627579689026 (0.23860197642445563) + | > loss_0: 2.7048745155334473 (2.738478866577147) + | > grad_norm_0: tensor(8.0737, device='cuda:0') (tensor(10.0216, device='cuda:0')) + | > loss_gen: 1.7879811525344849 (1.8832591536045074) + | > loss_kl: 2.5846848487854004 (2.511520478725435) + | > loss_feat: 3.119371175765991 (3.351884747028351) + | > loss_mel: 21.37281608581543 (21.28346037673951) + | > loss_duration: 1.4070546627044678 (1.3720202400684356) + | > amp_scaler: 512.0 (561.1520000000005) + | > loss_1: 30.271907806396484 (30.40214507293701) + | > grad_norm_1: tensor(59.8478, device='cuda:0') (tensor(121.8862, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 1.0406 (0.9056869754791262) + | > loader_time: 0.0221 (0.014397311210632327) + + + --> TIME: 2024-06-25 07:01:50 -- STEP: 525/591 -- GLOBAL_STEP: 15300 + | > loss_disc: 2.7155649662017822 (2.7382262920197977) + | > loss_disc_real_0: 0.32891908288002014 (0.21576369191919054) + | > loss_disc_real_1: 0.19533805549144745 (0.22916088240487234) + | > loss_disc_real_2: 0.22221510112285614 (0.23555768799214136) + | > loss_disc_real_3: 0.26804468035697937 (0.23594064874308449) + | > loss_disc_real_4: 0.21003088355064392 (0.23393964236690887) + | > loss_disc_real_5: 0.2279367595911026 (0.23851764380931853) + | > loss_0: 2.7155649662017822 (2.7382262920197977) + | > grad_norm_0: tensor(18.8450, device='cuda:0') (tensor(10.0524, device='cuda:0')) + | > loss_gen: 1.8579902648925781 (1.883237363270351) + | > loss_kl: 2.4179933071136475 (2.5093622802552735) + | > loss_feat: 3.4745287895202637 (3.34848849432809) + | > loss_mel: 21.586957931518555 (21.273961603073854) + | > loss_duration: 1.4061651229858398 (1.3736090746380034) + | > amp_scaler: 512.0 (558.811428571429) + | > loss_1: 30.743637084960938 (30.388658861432756) + | > grad_norm_1: tensor(251.8581, device='cuda:0') (tensor(124.0105, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 1.0585 (0.9135846246991841) + | > loader_time: 0.0178 (0.01471020335242862) + + + --> TIME: 2024-06-25 07:02:20 -- STEP: 550/591 -- GLOBAL_STEP: 15325 + | > loss_disc: 2.777750015258789 (2.738431813066655) + | > loss_disc_real_0: 0.25905370712280273 (0.2158855560692874) + | > loss_disc_real_1: 0.22587238252162933 (0.22913612596013333) + | > loss_disc_real_2: 0.22663091123104095 (0.2355693354660815) + | > loss_disc_real_3: 0.21278329193592072 (0.23595812816511502) + | > loss_disc_real_4: 0.23322007060050964 (0.23394018601287495) + | > loss_disc_real_5: 0.24380426108837128 (0.23854653282599014) + | > loss_0: 2.777750015258789 (2.738431813066655) + | > grad_norm_0: tensor(10.8240, device='cuda:0') (tensor(10.0892, device='cuda:0')) + | > loss_gen: 1.751550555229187 (1.8827933504364707) + | > loss_kl: 2.3539681434631348 (2.5068036703629937) + | > loss_feat: 2.9478073120117188 (3.3424515867233278) + | > loss_mel: 20.796838760375977 (21.262585463090385) + | > loss_duration: 1.3941402435302734 (1.3750759933211583) + | > amp_scaler: 512.0 (556.6836363636369) + | > loss_1: 29.244304656982422 (30.369710107283158) + | > grad_norm_1: tensor(89.9529, device='cuda:0') (tensor(123.4303, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 1.1957 (0.9251744070920078) + | > loader_time: 0.0277 (0.015389276851307263) + + + --> TIME: 2024-06-25 07:02:50 -- STEP: 575/591 -- GLOBAL_STEP: 15350 + | > loss_disc: 2.749682664871216 (2.738652537801991) + | > loss_disc_real_0: 0.22188764810562134 (0.21594441084758095) + | > loss_disc_real_1: 0.21457017958164215 (0.2289856748736423) + | > loss_disc_real_2: 0.24663451313972473 (0.23550449985524888) + | > loss_disc_real_3: 0.24059763550758362 (0.23600848607395006) + | > loss_disc_real_4: 0.22896747291088104 (0.23392043974088586) + | > loss_disc_real_5: 0.24743658304214478 (0.23861331690912663) + | > loss_0: 2.749682664871216 (2.738652537801991) + | > grad_norm_0: tensor(4.2007, device='cuda:0') (tensor(10.1304, device='cuda:0')) + | > loss_gen: 1.9026141166687012 (1.8818600281425144) + | > loss_kl: 2.5144102573394775 (2.504362474939099) + | > loss_feat: 3.078529119491577 (3.3322985620084014) + | > loss_mel: 21.29150390625 (21.24741328695547) + | > loss_duration: 1.4303536415100098 (1.3765936179783032) + | > amp_scaler: 512.0 (554.740869565218) + | > loss_1: 30.217411041259766 (30.342528003195056) + | > grad_norm_1: tensor(79.5476, device='cuda:0') (tensor(123.3959, device='cuda:0')) + | > current_lr_0: 0.0001993759366021797 + | > current_lr_1: 0.0001993759366021797 + | > step_time: 1.2796 (0.9356856939067011) + | > loader_time: 0.0229 (0.015774941237076477) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036768507450184924 (-8.111304425178524e-06) + | > avg_loss_disc: 2.781233346208613 (+0.06589281305353722) + | > avg_loss_disc_real_0: 0.1548473698661683 (-0.11808237179796752) + | > avg_loss_disc_real_1: 0.20926511573030593 (+0.0014661991215766945) + | > avg_loss_disc_real_2: 0.23217333789835584 (+0.023627995810610175) + | > avg_loss_disc_real_3: 0.20033282865869237 (-0.014570661998809659) + | > avg_loss_disc_real_4: 0.19711969380683086 (-0.027461577286111566) + | > avg_loss_disc_real_5: 0.21828172340037974 (-0.017282262127450182) + | > avg_loss_0: 2.781233346208613 (+0.06589281305353722) + | > avg_loss_gen: 1.5949407663751156 (-0.2722267977734829) + | > avg_loss_kl: 2.593221030336744 (+0.0001888528783258181) + | > avg_loss_feat: 3.3283624141774277 (+0.0216008957396161) + | > avg_loss_mel: 21.15271426261739 (-0.15065217525400598) + | > avg_loss_duration: 1.4077749024046229 (+0.03384515326073845) + | > avg_loss_1: 30.077013259238385 (-0.36724423347635593) + + + > EPOCH: 27/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 07:03:18)  + + --> TIME: 2024-06-25 07:03:26 -- STEP: 9/591 -- GLOBAL_STEP: 15375 + | > loss_disc: 2.7889199256896973 (2.753875494003296) + | > loss_disc_real_0: 0.29821571707725525 (0.232695781522327) + | > loss_disc_real_1: 0.23565667867660522 (0.2329018215338389) + | > loss_disc_real_2: 0.2934393882751465 (0.2398033406999376) + | > loss_disc_real_3: 0.3049773573875427 (0.24681375920772552) + | > loss_disc_real_4: 0.2384440302848816 (0.2403265121910307) + | > loss_disc_real_5: 0.257918119430542 (0.24292350146505567) + | > loss_0: 2.7889199256896973 (2.753875494003296) + | > grad_norm_0: tensor(14.3846, device='cuda:0') (tensor(15.2136, device='cuda:0')) + | > loss_gen: 1.851932406425476 (1.9005046950446234) + | > loss_kl: 2.7148852348327637 (2.70289585325453) + | > loss_feat: 3.6042261123657227 (3.5709625085194907) + | > loss_mel: 21.226839065551758 (21.410432391696506) + | > loss_duration: 1.293853521347046 (1.3492171102099948) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.691736221313477 (30.934012095133465) + | > grad_norm_1: tensor(148.6490, device='cuda:0') (tensor(145.0889, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.7628 (0.7651796076032851) + | > loader_time: 0.0088 (0.008455779817369249) + + + --> TIME: 2024-06-25 07:03:46 -- STEP: 34/591 -- GLOBAL_STEP: 15400 + | > loss_disc: 2.771084785461426 (2.743193843785454) + | > loss_disc_real_0: 0.20266781747341156 (0.21956445495871937) + | > loss_disc_real_1: 0.28863978385925293 (0.2297534903182703) + | > loss_disc_real_2: 0.21422970294952393 (0.23373751035507986) + | > loss_disc_real_3: 0.25514063239097595 (0.23693100450670018) + | > loss_disc_real_4: 0.23865991830825806 (0.23731305055758534) + | > loss_disc_real_5: 0.23598796129226685 (0.24124143053503597) + | > loss_0: 2.771084785461426 (2.743193843785454) + | > grad_norm_0: tensor(7.0164, device='cuda:0') (tensor(11.7609, device='cuda:0')) + | > loss_gen: 1.9952808618545532 (1.8962846398353577) + | > loss_kl: 2.6834213733673096 (2.654472168754129) + | > loss_feat: 3.377507209777832 (3.5451826277901146) + | > loss_mel: 21.066598892211914 (21.508081997142117) + | > loss_duration: 1.3785631656646729 (1.3462978566394135) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.501371383666992 (30.950319065767175) + | > grad_norm_1: tensor(63.8875, device='cuda:0') (tensor(109.0568, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.7893 (0.7833744497860179) + | > loader_time: 0.0101 (0.009503553895389332) + + + --> TIME: 2024-06-25 07:04:07 -- STEP: 59/591 -- GLOBAL_STEP: 15425 + | > loss_disc: 2.791867733001709 (2.740935850951631) + | > loss_disc_real_0: 0.2885821461677551 (0.21787451056100554) + | > loss_disc_real_1: 0.23975227773189545 (0.22969263968831402) + | > loss_disc_real_2: 0.23704615235328674 (0.2343379006547443) + | > loss_disc_real_3: 0.20349600911140442 (0.23575586661443873) + | > loss_disc_real_4: 0.23570631444454193 (0.23617496581400854) + | > loss_disc_real_5: 0.2323087751865387 (0.23957768638255234) + | > loss_0: 2.791867733001709 (2.740935850951631) + | > grad_norm_0: tensor(9.6288, device='cuda:0') (tensor(11.2203, device='cuda:0')) + | > loss_gen: 1.8517138957977295 (1.8879744905536457) + | > loss_kl: 2.4845101833343506 (2.597903999231629) + | > loss_feat: 3.305720329284668 (3.499383005045228) + | > loss_mel: 20.978124618530273 (21.408446328114653) + | > loss_duration: 1.340238332748413 (1.3533119912874902) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.960308074951172 (30.747019654613432) + | > grad_norm_1: tensor(131.5107, device='cuda:0') (tensor(108.8063, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.8198 (0.7987932916414939) + | > loader_time: 0.0101 (0.009888863159438313) + + + --> TIME: 2024-06-25 07:04:28 -- STEP: 84/591 -- GLOBAL_STEP: 15450 + | > loss_disc: 2.7468721866607666 (2.7389613900865823) + | > loss_disc_real_0: 0.29992538690567017 (0.21732821900929725) + | > loss_disc_real_1: 0.2161380648612976 (0.23071143137557165) + | > loss_disc_real_2: 0.26736214756965637 (0.2341502048075199) + | > loss_disc_real_3: 0.2149597555398941 (0.23595442747076353) + | > loss_disc_real_4: 0.28363484144210815 (0.23590191932661192) + | > loss_disc_real_5: 0.22175483405590057 (0.2386220306867645) + | > loss_0: 2.7468721866607666 (2.7389613900865823) + | > grad_norm_0: tensor(12.9172, device='cuda:0') (tensor(10.8314, device='cuda:0')) + | > loss_gen: 1.733144760131836 (1.8873813194887978) + | > loss_kl: 2.5874264240264893 (2.5830910007158914) + | > loss_feat: 3.593919515609741 (3.4797159830729165) + | > loss_mel: 21.77851676940918 (21.409575643993563) + | > loss_duration: 1.3393936157226562 (1.3590585220427738) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.032400131225586 (30.71882234300886) + | > grad_norm_1: tensor(194.1464, device='cuda:0') (tensor(117.8992, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.8458 (0.8081673099881127) + | > loader_time: 0.0118 (0.010304780233474009) + + + --> TIME: 2024-06-25 07:04:49 -- STEP: 109/591 -- GLOBAL_STEP: 15475 + | > loss_disc: 2.738415241241455 (2.746716678689378) + | > loss_disc_real_0: 0.14598795771598816 (0.2167664836032675) + | > loss_disc_real_1: 0.21619124710559845 (0.2303794037311449) + | > loss_disc_real_2: 0.25212568044662476 (0.2345132334243267) + | > loss_disc_real_3: 0.2735011875629425 (0.23841572546083992) + | > loss_disc_real_4: 0.22216273844242096 (0.23554198662622258) + | > loss_disc_real_5: 0.2336052805185318 (0.23847431504945144) + | > loss_0: 2.738415241241455 (2.746716678689378) + | > grad_norm_0: tensor(14.7513, device='cuda:0') (tensor(11.0372, device='cuda:0')) + | > loss_gen: 2.0177130699157715 (1.8843169781046176) + | > loss_kl: 2.550231456756592 (2.563434769254211) + | > loss_feat: 3.5612189769744873 (3.451150257653053) + | > loss_mel: 22.125595092773438 (21.42274307110988) + | > loss_duration: 1.3881542682647705 (1.3660322504306057) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.642913818359375 (30.68767720843674) + | > grad_norm_1: tensor(147.0878, device='cuda:0') (tensor(110.9005, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.8337 (0.8146432692851495) + | > loader_time: 0.0117 (0.01054832257262064) + + + --> TIME: 2024-06-25 07:05:11 -- STEP: 134/591 -- GLOBAL_STEP: 15500 + | > loss_disc: 2.7657268047332764 (2.745213841324421) + | > loss_disc_real_0: 0.12080446630716324 (0.21594062764475594) + | > loss_disc_real_1: 0.20538949966430664 (0.23002712161683325) + | > loss_disc_real_2: 0.19713562726974487 (0.23367711579176917) + | > loss_disc_real_3: 0.23340079188346863 (0.2389535436879343) + | > loss_disc_real_4: 0.21167102456092834 (0.23560312623853114) + | > loss_disc_real_5: 0.2508895695209503 (0.23866685324195605) + | > loss_0: 2.7657268047332764 (2.745213841324421) + | > grad_norm_0: tensor(22.0768, device='cuda:0') (tensor(10.4281, device='cuda:0')) + | > loss_gen: 2.1337246894836426 (1.8819776915792208) + | > loss_kl: 2.4269816875457764 (2.5541871537023506) + | > loss_feat: 3.657921552658081 (3.4437924748036397) + | > loss_mel: 21.414337158203125 (21.425983471656917) + | > loss_duration: 1.356115460395813 (1.36729277692624) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.98908042907715 (30.67323344501097) + | > grad_norm_1: tensor(125.0827, device='cuda:0') (tensor(111.8779, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.8281 (0.8203969962561308) + | > loader_time: 0.0123 (0.010732535106032646) + + + --> TIME: 2024-06-25 07:05:33 -- STEP: 159/591 -- GLOBAL_STEP: 15525 + | > loss_disc: 2.7493081092834473 (2.744139936735044) + | > loss_disc_real_0: 0.2836453318595886 (0.21630872036301116) + | > loss_disc_real_1: 0.20261268317699432 (0.2296716608341385) + | > loss_disc_real_2: 0.2424425333738327 (0.23425905031603086) + | > loss_disc_real_3: 0.23244981467723846 (0.23930417638529772) + | > loss_disc_real_4: 0.25713014602661133 (0.23530575220689834) + | > loss_disc_real_5: 0.263914555311203 (0.23866298091861438) + | > loss_0: 2.7493081092834473 (2.744139936735044) + | > grad_norm_0: tensor(13.1165, device='cuda:0') (tensor(10.5743, device='cuda:0')) + | > loss_gen: 1.7406988143920898 (1.88251704389944) + | > loss_kl: 2.571424722671509 (2.5519067251457352) + | > loss_feat: 3.1548001766204834 (3.445645252863566) + | > loss_mel: 20.664581298828125 (21.42288422734483) + | > loss_duration: 1.3570328950881958 (1.3653498003317879) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.488536834716797 (30.66830293187555) + | > grad_norm_1: tensor(212.6902, device='cuda:0') (tensor(111.0526, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.8516 (0.8259920204210583) + | > loader_time: 0.0117 (0.010900777840764266) + + + --> TIME: 2024-06-25 07:05:55 -- STEP: 184/591 -- GLOBAL_STEP: 15550 + | > loss_disc: 2.724898338317871 (2.7426407920277627) + | > loss_disc_real_0: 0.17616143822669983 (0.21578459158215835) + | > loss_disc_real_1: 0.2402457892894745 (0.22956199045090572) + | > loss_disc_real_2: 0.2193266600370407 (0.23401918132667957) + | > loss_disc_real_3: 0.24176262319087982 (0.23931066011605057) + | > loss_disc_real_4: 0.20985190570354462 (0.23492136962063934) + | > loss_disc_real_5: 0.2117287963628769 (0.23871576373019945) + | > loss_0: 2.724898338317871 (2.7426407920277627) + | > grad_norm_0: tensor(9.4627, device='cuda:0') (tensor(10.6985, device='cuda:0')) + | > loss_gen: 2.025131940841675 (1.8847070005924806) + | > loss_kl: 2.4538824558258057 (2.546658153119295) + | > loss_feat: 3.559931755065918 (3.4437799220499783) + | > loss_mel: 21.799657821655273 (21.430554483247853) + | > loss_duration: 1.3694102764129639 (1.3634432528329932) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.2080135345459 (30.669142702351447) + | > grad_norm_1: tensor(70.9677, device='cuda:0') (tensor(110.5705, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.8603 (0.8312348231025366) + | > loader_time: 0.0117 (0.011069888653962506) + + + --> TIME: 2024-06-25 07:06:17 -- STEP: 209/591 -- GLOBAL_STEP: 15575 + | > loss_disc: 2.7040107250213623 (2.7406903043318014) + | > loss_disc_real_0: 0.1783638298511505 (0.21524181220520056) + | > loss_disc_real_1: 0.21737317740917206 (0.22945280537080537) + | > loss_disc_real_2: 0.24562479555606842 (0.2341682066044739) + | > loss_disc_real_3: 0.25475114583969116 (0.23895546900883816) + | > loss_disc_real_4: 0.24420218169689178 (0.23500112454856983) + | > loss_disc_real_5: 0.2620076537132263 (0.2388267865734237) + | > loss_0: 2.7040107250213623 (2.7406903043318014) + | > grad_norm_0: tensor(7.7612, device='cuda:0') (tensor(10.5110, device='cuda:0')) + | > loss_gen: 1.8702468872070312 (1.8852164790961161) + | > loss_kl: 2.5745975971221924 (2.5447168167698324) + | > loss_feat: 3.339517593383789 (3.441561868886628) + | > loss_mel: 21.301406860351562 (21.44886776134729) + | > loss_duration: 1.3416869640350342 (1.3627429898275714) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.42745590209961 (30.68310586117101) + | > grad_norm_1: tensor(113.2358, device='cuda:0') (tensor(110.5840, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.8676 (0.8361425046145062) + | > loader_time: 0.0153 (0.011343774613010824) + + + --> TIME: 2024-06-25 07:06:40 -- STEP: 234/591 -- GLOBAL_STEP: 15600 + | > loss_disc: 2.731903553009033 (2.739200863063844) + | > loss_disc_real_0: 0.16280460357666016 (0.21487104408761376) + | > loss_disc_real_1: 0.21661368012428284 (0.22907666588186198) + | > loss_disc_real_2: 0.24279387295246124 (0.2342472405005724) + | > loss_disc_real_3: 0.23300302028656006 (0.2385620024596524) + | > loss_disc_real_4: 0.24721404910087585 (0.23493140337304172) + | > loss_disc_real_5: 0.26176074147224426 (0.2389169439800784) + | > loss_0: 2.731903553009033 (2.739200863063844) + | > grad_norm_0: tensor(10.0941, device='cuda:0') (tensor(10.4986, device='cuda:0')) + | > loss_gen: 2.0002002716064453 (1.8860280697162335) + | > loss_kl: 2.6809921264648438 (2.5409718884362116) + | > loss_feat: 3.5302605628967285 (3.438549403451447) + | > loss_mel: 21.198131561279297 (21.44111146682349) + | > loss_duration: 1.3808741569519043 (1.3624158904083774) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.79045867919922 (30.669076634268475) + | > grad_norm_1: tensor(73.3503, device='cuda:0') (tensor(109.8702, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.8808 (0.8408587325332513) + | > loader_time: 0.0159 (0.011644638501680812) + + + --> TIME: 2024-06-25 07:07:02 -- STEP: 259/591 -- GLOBAL_STEP: 15625 + | > loss_disc: 2.758361577987671 (2.737609743611692) + | > loss_disc_real_0: 0.20613756775856018 (0.2147174648451529) + | > loss_disc_real_1: 0.23828621208667755 (0.22896771380339812) + | > loss_disc_real_2: 0.2269260287284851 (0.23416474104848148) + | > loss_disc_real_3: 0.2360822558403015 (0.23826727708334167) + | > loss_disc_real_4: 0.258830189704895 (0.2347590181703273) + | > loss_disc_real_5: 0.25383538007736206 (0.2388888839128855) + | > loss_0: 2.758361577987671 (2.737609743611692) + | > grad_norm_0: tensor(4.9128, device='cuda:0') (tensor(10.4212, device='cuda:0')) + | > loss_gen: 1.9282817840576172 (1.8856584551711801) + | > loss_kl: 2.5945911407470703 (2.5369362904758526) + | > loss_feat: 3.3705601692199707 (3.4376080459609457) + | > loss_mel: 21.237895965576172 (21.430120158839877) + | > loss_duration: 1.3891551494598389 (1.3624495281676068) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.520483016967773 (30.65277235112135) + | > grad_norm_1: tensor(54.7404, device='cuda:0') (tensor(108.7154, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.8965 (0.8460864569689778) + | > loader_time: 0.0137 (0.011939530206923317) + + + --> TIME: 2024-06-25 07:07:25 -- STEP: 284/591 -- GLOBAL_STEP: 15650 + | > loss_disc: 2.8089957237243652 (2.738259925808704) + | > loss_disc_real_0: 0.3429079055786133 (0.21482148554018687) + | > loss_disc_real_1: 0.2513723373413086 (0.22898506304957497) + | > loss_disc_real_2: 0.25163763761520386 (0.23470011127876564) + | > loss_disc_real_3: 0.27857938408851624 (0.23818274199123113) + | > loss_disc_real_4: 0.22223101556301117 (0.23467341353985624) + | > loss_disc_real_5: 0.26152923703193665 (0.2388056076433457) + | > loss_0: 2.8089957237243652 (2.738259925808704) + | > grad_norm_0: tensor(19.0640, device='cuda:0') (tensor(10.4972, device='cuda:0')) + | > loss_gen: 1.808424949645996 (1.885567007770001) + | > loss_kl: 2.462282657623291 (2.531293564279314) + | > loss_feat: 3.209699869155884 (3.4324210621941256) + | > loss_mel: 21.339584350585938 (21.41099058070654) + | > loss_duration: 1.3838450908660889 (1.3629362633530522) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.20383644104004 (30.62320836161224) + | > grad_norm_1: tensor(248.0817, device='cuda:0') (tensor(112.0017, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.8928 (0.8506535098586284) + | > loader_time: 0.0141 (0.012219131832391445) + + + --> TIME: 2024-06-25 07:07:49 -- STEP: 309/591 -- GLOBAL_STEP: 15675 + | > loss_disc: 2.8664464950561523 (2.7398344035287496) + | > loss_disc_real_0: 0.15937469899654388 (0.21465352122162537) + | > loss_disc_real_1: 0.2773127853870392 (0.22931514516424592) + | > loss_disc_real_2: 0.2909700572490692 (0.23518790198568387) + | > loss_disc_real_3: 0.2683074474334717 (0.2379696949303729) + | > loss_disc_real_4: 0.24843242764472961 (0.23474749001677367) + | > loss_disc_real_5: 0.2246488332748413 (0.23880343602120296) + | > loss_0: 2.8664464950561523 (2.7398344035287496) + | > grad_norm_0: tensor(16.2757, device='cuda:0') (tensor(10.5464, device='cuda:0')) + | > loss_gen: 1.8175069093704224 (1.8857462907686202) + | > loss_kl: 2.4157943725585938 (2.528709660841808) + | > loss_feat: 3.4469153881073 (3.42746627292201) + | > loss_mel: 20.684579849243164 (21.395665789113462) + | > loss_duration: 1.3817185163497925 (1.3632926944775872) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.74651527404785 (30.600880607432146) + | > grad_norm_1: tensor(97.9617, device='cuda:0') (tensor(112.6318, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.934 (0.8560661922380762) + | > loader_time: 0.0158 (0.012448073976634003) + + + --> TIME: 2024-06-25 07:08:13 -- STEP: 334/591 -- GLOBAL_STEP: 15700 + | > loss_disc: 2.813725233078003 (2.7405396821256156) + | > loss_disc_real_0: 0.262029230594635 (0.2146398415510169) + | > loss_disc_real_1: 0.2331489771604538 (0.22938598585342931) + | > loss_disc_real_2: 0.21358665823936462 (0.23537231076084925) + | > loss_disc_real_3: 0.22084395587444305 (0.23768549515101725) + | > loss_disc_real_4: 0.2325892597436905 (0.23481353662328092) + | > loss_disc_real_5: 0.22376331686973572 (0.23898918578724662) + | > loss_0: 2.813725233078003 (2.7405396821256156) + | > grad_norm_0: tensor(9.0616, device='cuda:0') (tensor(10.4582, device='cuda:0')) + | > loss_gen: 1.8146040439605713 (1.883102325622193) + | > loss_kl: 2.5690438747406006 (2.5260126383718604) + | > loss_feat: 3.1618878841400146 (3.414936788067846) + | > loss_mel: 21.124849319458008 (21.3859516931865) + | > loss_duration: 1.3881100416183472 (1.3642031664619896) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.058494567871094 (30.574206517842) + | > grad_norm_1: tensor(186.2981, device='cuda:0') (tensor(113.4561, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.9484 (0.8615106137212879) + | > loader_time: 0.0139 (0.012696603101170706) + + + --> TIME: 2024-06-25 07:08:37 -- STEP: 359/591 -- GLOBAL_STEP: 15725 + | > loss_disc: 2.76185941696167 (2.740757802734799) + | > loss_disc_real_0: 0.2552770674228668 (0.21473750752243823) + | > loss_disc_real_1: 0.289059579372406 (0.2294116683491093) + | > loss_disc_real_2: 0.2649262845516205 (0.2356867085841373) + | > loss_disc_real_3: 0.21070964634418488 (0.23742589034907996) + | > loss_disc_real_4: 0.25584495067596436 (0.23497660113576396) + | > loss_disc_real_5: 0.2748434245586395 (0.2390933114007987) + | > loss_0: 2.76185941696167 (2.740757802734799) + | > grad_norm_0: tensor(9.1332, device='cuda:0') (tensor(10.3495, device='cuda:0')) + | > loss_gen: 1.7782247066497803 (1.8819317435819791) + | > loss_kl: 2.573474884033203 (2.5246639995521805) + | > loss_feat: 3.350994825363159 (3.4047826365840135) + | > loss_mel: 20.78912925720215 (21.36735377935977) + | > loss_duration: 1.3634097576141357 (1.365549124050936) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.855234146118164 (30.54428121306438) + | > grad_norm_1: tensor(187.3393, device='cuda:0') (tensor(119.6785, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.9342 (0.8668025994367254) + | > loader_time: 0.0175 (0.01295646053834878) + + + --> TIME: 2024-06-25 07:09:01 -- STEP: 384/591 -- GLOBAL_STEP: 15750 + | > loss_disc: 2.8012657165527344 (2.740673877298832) + | > loss_disc_real_0: 0.28757405281066895 (0.2147094612203849) + | > loss_disc_real_1: 0.23394304513931274 (0.22939792935115597) + | > loss_disc_real_2: 0.2553812861442566 (0.2355950855029126) + | > loss_disc_real_3: 0.26168733835220337 (0.2375065265223384) + | > loss_disc_real_4: 0.24378471076488495 (0.23490021765852967) + | > loss_disc_real_5: 0.26002150774002075 (0.23897321813274175) + | > loss_0: 2.8012657165527344 (2.740673877298832) + | > grad_norm_0: tensor(15.4940, device='cuda:0') (tensor(10.3463, device='cuda:0')) + | > loss_gen: 1.7605862617492676 (1.8819593364993732) + | > loss_kl: 2.35990571975708 (2.521988388150929) + | > loss_feat: 2.9540510177612305 (3.400394936402639) + | > loss_mel: 21.04538345336914 (21.362108618021) + | > loss_duration: 1.4158886671066284 (1.36718151376893) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.53581428527832 (30.533632735411327) + | > grad_norm_1: tensor(101.8078, device='cuda:0') (tensor(122.8647, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.9678 (0.8727507709215084) + | > loader_time: 0.0185 (0.013188843304912249) + + + --> TIME: 2024-06-25 07:09:26 -- STEP: 409/591 -- GLOBAL_STEP: 15775 + | > loss_disc: 2.7886228561401367 (2.739529345612654) + | > loss_disc_real_0: 0.06307446211576462 (0.21439141661730254) + | > loss_disc_real_1: 0.20571495592594147 (0.22930408810199326) + | > loss_disc_real_2: 0.21379929780960083 (0.2353902222329072) + | > loss_disc_real_3: 0.20204082131385803 (0.23710389078246352) + | > loss_disc_real_4: 0.19388613104820251 (0.23486629033729908) + | > loss_disc_real_5: 0.2170683741569519 (0.23895415077145352) + | > loss_0: 2.7886228561401367 (2.739529345612654) + | > grad_norm_0: tensor(38.7227, device='cuda:0') (tensor(10.4267, device='cuda:0')) + | > loss_gen: 2.1386637687683105 (1.882265972624781) + | > loss_kl: 2.531099557876587 (2.5202256706349715) + | > loss_feat: 3.3605666160583496 (3.397703000560658) + | > loss_mel: 21.101459503173828 (21.357280176251315) + | > loss_duration: 1.3520455360412598 (1.367911390978432) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.483835220336914 (30.525386162084004) + | > grad_norm_1: tensor(172.0708, device='cuda:0') (tensor(123.9510, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.9717 (0.878455968241237) + | > loader_time: 0.0181 (0.013456362092407526) + + + --> TIME: 2024-06-25 07:09:51 -- STEP: 434/591 -- GLOBAL_STEP: 15800 + | > loss_disc: 2.7193820476531982 (2.7385954757989275) + | > loss_disc_real_0: 0.26735246181488037 (0.21465962177597434) + | > loss_disc_real_1: 0.2145429253578186 (0.22913928127371222) + | > loss_disc_real_2: 0.23449189960956573 (0.2352858818552461) + | > loss_disc_real_3: 0.2351604551076889 (0.23700807140582167) + | > loss_disc_real_4: 0.2055191546678543 (0.23476303729318804) + | > loss_disc_real_5: 0.22205907106399536 (0.23886152480467124) + | > loss_0: 2.7193820476531982 (2.7385954757989275) + | > grad_norm_0: tensor(9.6142, device='cuda:0') (tensor(10.4297, device='cuda:0')) + | > loss_gen: 1.9402592182159424 (1.8823684239717122) + | > loss_kl: 2.5524544715881348 (2.517617431104458) + | > loss_feat: 3.503389596939087 (3.396405890240647) + | > loss_mel: 21.594209671020508 (21.339603155988694) + | > loss_duration: 1.3691645860671997 (1.3688603634109135) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.95948028564453 (30.50485522186701) + | > grad_norm_1: tensor(85.5615, device='cuda:0') (tensor(124.2987, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 0.9982 (0.8849129946001114) + | > loader_time: 0.0193 (0.013687927052722) + + + --> TIME: 2024-06-25 07:10:17 -- STEP: 459/591 -- GLOBAL_STEP: 15825 + | > loss_disc: 2.7274551391601562 (2.7385626680710735) + | > loss_disc_real_0: 0.24702191352844238 (0.2144019194028476) + | > loss_disc_real_1: 0.2281043529510498 (0.22913996885025423) + | > loss_disc_real_2: 0.2354462891817093 (0.23545048640704103) + | > loss_disc_real_3: 0.25679609179496765 (0.23704791202134817) + | > loss_disc_real_4: 0.23976579308509827 (0.2347625914092677) + | > loss_disc_real_5: 0.2528362572193146 (0.23891873401234612) + | > loss_0: 2.7274551391601562 (2.7385626680710735) + | > grad_norm_0: tensor(6.6657, device='cuda:0') (tensor(10.3568, device='cuda:0')) + | > loss_gen: 1.849035382270813 (1.8820554082949437) + | > loss_kl: 2.5350265502929688 (2.514082997452979) + | > loss_feat: 3.243586540222168 (3.389684048093742) + | > loss_mel: 21.81488037109375 (21.323719685373735) + | > loss_duration: 1.3995565176010132 (1.3698210742219072) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.842086791992188 (30.479363169285726) + | > grad_norm_1: tensor(116.6952, device='cuda:0') (tensor(124.1930, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 1.0155 (0.8918315335816028) + | > loader_time: 0.0174 (0.013963693627085301) + + + --> TIME: 2024-06-25 07:10:43 -- STEP: 484/591 -- GLOBAL_STEP: 15850 + | > loss_disc: 2.763474464416504 (2.7400436263439083) + | > loss_disc_real_0: 0.28373080492019653 (0.21475883687207523) + | > loss_disc_real_1: 0.28143054246902466 (0.22938147762959654) + | > loss_disc_real_2: 0.261362224817276 (0.235731342945956) + | > loss_disc_real_3: 0.20779508352279663 (0.2370291369820922) + | > loss_disc_real_4: 0.22633899748325348 (0.23482278789863115) + | > loss_disc_real_5: 0.23636622726917267 (0.23904241978629562) + | > loss_0: 2.763474464416504 (2.7400436263439083) + | > grad_norm_0: tensor(9.9688, device='cuda:0') (tensor(10.5384, device='cuda:0')) + | > loss_gen: 1.8140324354171753 (1.8823257435944454) + | > loss_kl: 2.5129191875457764 (2.5124014433750435) + | > loss_feat: 3.2388556003570557 (3.3811626907222525) + | > loss_mel: 20.813554763793945 (21.317740357611783) + | > loss_duration: 1.4007225036621094 (1.3710786215037345) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.78008460998535 (30.46470880902503) + | > grad_norm_1: tensor(90.0686, device='cuda:0') (tensor(125.4674, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 1.0222 (0.8985421820120378) + | > loader_time: 0.0217 (0.01426509392162985) + + + --> TIME: 2024-06-25 07:11:10 -- STEP: 509/591 -- GLOBAL_STEP: 15875 + | > loss_disc: 2.746689796447754 (2.740179783234662) + | > loss_disc_real_0: 0.23366375267505646 (0.21459179138856224) + | > loss_disc_real_1: 0.24020981788635254 (0.22936633308299165) + | > loss_disc_real_2: 0.2249438613653183 (0.23577307000726988) + | > loss_disc_real_3: 0.2699912488460541 (0.2370702192802092) + | > loss_disc_real_4: 0.21884974837303162 (0.23486528617342703) + | > loss_disc_real_5: 0.2293449342250824 (0.23903174016236792) + | > loss_0: 2.746689796447754 (2.740179783234662) + | > grad_norm_0: tensor(6.4973, device='cuda:0') (tensor(10.3650, device='cuda:0')) + | > loss_gen: 1.8297820091247559 (1.8811892249963607) + | > loss_kl: 2.4774374961853027 (2.5115142711721843) + | > loss_feat: 3.212423086166382 (3.3740288963018092) + | > loss_mel: 21.0133056640625 (21.301634078409904) + | > loss_duration: 1.40897536277771 (1.3724425935089462) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.941925048828125 (30.440809017548627) + | > grad_norm_1: tensor(67.7534, device='cuda:0') (tensor(124.5051, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 1.0321 (0.9068988311033118) + | > loader_time: 0.0212 (0.014574206882469312) + + + --> TIME: 2024-06-25 07:11:39 -- STEP: 534/591 -- GLOBAL_STEP: 15900 + | > loss_disc: 2.798992872238159 (2.740391149503015) + | > loss_disc_real_0: 0.19352726638317108 (0.2144263268968139) + | > loss_disc_real_1: 0.2561229467391968 (0.2294744233625212) + | > loss_disc_real_2: 0.22152817249298096 (0.23587871959593418) + | > loss_disc_real_3: 0.2189890742301941 (0.2369863676946708) + | > loss_disc_real_4: 0.24182136356830597 (0.23490521861484434) + | > loss_disc_real_5: 0.26015883684158325 (0.2390926682882095) + | > loss_0: 2.798992872238159 (2.740391149503015) + | > grad_norm_0: tensor(6.1758, device='cuda:0') (tensor(10.2597, device='cuda:0')) + | > loss_gen: 1.7882740497589111 (1.8800316888294866) + | > loss_kl: 2.380350351333618 (2.508294059096204) + | > loss_feat: 3.0087883472442627 (3.3647044839037488) + | > loss_mel: 20.053266525268555 (21.281096558445842) + | > loss_duration: 1.4046024084091187 (1.3739544738098022) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 28.635282516479492 (30.408081208275497) + | > grad_norm_1: tensor(132.1347, device='cuda:0') (tensor(122.9677, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 1.1232 (0.9166190320632878) + | > loader_time: 0.0238 (0.01489413007814786) + + + --> TIME: 2024-06-25 07:12:09 -- STEP: 559/591 -- GLOBAL_STEP: 15925 + | > loss_disc: 2.798348903656006 (2.74065847558924) + | > loss_disc_real_0: 0.15567396581172943 (0.21445165683843578) + | > loss_disc_real_1: 0.2287171334028244 (0.22949962184659484) + | > loss_disc_real_2: 0.22172196209430695 (0.23599071392859464) + | > loss_disc_real_3: 0.26448312401771545 (0.23696354724640065) + | > loss_disc_real_4: 0.25217878818511963 (0.2348689273239675) + | > loss_disc_real_5: 0.23972350358963013 (0.23907755811120615) + | > loss_0: 2.798348903656006 (2.74065847558924) + | > grad_norm_0: tensor(13.0333, device='cuda:0') (tensor(10.2436, device='cuda:0')) + | > loss_gen: 1.8149054050445557 (1.879558569203549) + | > loss_kl: 2.5213706493377686 (2.506190220230594) + | > loss_feat: 3.0426788330078125 (3.3577162050817004) + | > loss_mel: 21.107654571533203 (21.263637024089558) + | > loss_duration: 1.418637752532959 (1.3756374886296083) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.90524673461914 (30.382739461172033) + | > grad_norm_1: tensor(143.2957, device='cuda:0') (tensor(123.1913, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 1.142 (0.9278816293945038) + | > loader_time: 0.024 (0.015272823343976453) + + + --> TIME: 2024-06-25 07:12:40 -- STEP: 584/591 -- GLOBAL_STEP: 15950 + | > loss_disc: 2.735769033432007 (2.7406758633378434) + | > loss_disc_real_0: 0.1619211584329605 (0.21447850317273234) + | > loss_disc_real_1: 0.2004833221435547 (0.22948853794026047) + | > loss_disc_real_2: 0.24397951364517212 (0.23591134582378281) + | > loss_disc_real_3: 0.24039822816848755 (0.23682877189186333) + | > loss_disc_real_4: 0.22625543177127838 (0.23471374164195094) + | > loss_disc_real_5: 0.24589306116104126 (0.23900168919808243) + | > loss_0: 2.735769033432007 (2.7406758633378434) + | > grad_norm_0: tensor(8.0264, device='cuda:0') (tensor(10.3338, device='cuda:0')) + | > loss_gen: 2.0857372283935547 (1.8797562589792356) + | > loss_kl: 2.4132659435272217 (2.5040638299837514) + | > loss_feat: 3.2075634002685547 (3.352340424305772) + | > loss_mel: 20.626201629638672 (21.24600095944861) + | > loss_duration: 1.4022669792175293 (1.3771278298880958) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.735036849975586 (30.35928925749374) + | > grad_norm_1: tensor(62.5215, device='cuda:0') (tensor(122.6291, device='cuda:0')) + | > current_lr_0: 0.00019935101461010442 + | > current_lr_1: 0.00019935101461010442 + | > step_time: 1.2278 (0.9406264869317614) + | > loader_time: 0.0244 (0.015682182083391165) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036789102757230716 (+2.0595307045792426e-06) + | > avg_loss_disc: 2.7345516022215506 (-0.046681743987062596) + | > avg_loss_disc_real_0: 0.34422494788119135 (+0.18937757801502306) + | > avg_loss_disc_real_1: 0.24336594216367033 (+0.0341008264333644) + | > avg_loss_disc_real_2: 0.26513269828989156 (+0.032959360391535714) + | > avg_loss_disc_real_3: 0.22885026576671194 (+0.028517437108019572) + | > avg_loss_disc_real_4: 0.23346569500070938 (+0.03634600119387851) + | > avg_loss_disc_real_5: 0.23804759567088268 (+0.019765872270502943) + | > avg_loss_0: 2.7345516022215506 (-0.046681743987062596) + | > avg_loss_gen: 2.10120288869168 (+0.5062621223165644) + | > avg_loss_kl: 2.5854419444469694 (-0.007779085889774606) + | > avg_loss_feat: 3.2791213329802167 (-0.049241081197211045) + | > avg_loss_mel: 20.920513518313143 (-0.2322007443042473) + | > avg_loss_duration: 1.3872164335656674 (-0.020558468838955513) + | > avg_loss_1: 30.273496343734415 (+0.19648308449603036) + + + > EPOCH: 28/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 07:12:58)  + + --> TIME: 2024-06-25 07:13:13 -- STEP: 18/591 -- GLOBAL_STEP: 15975 + | > loss_disc: 2.7277259826660156 (2.7521318594614663) + | > loss_disc_real_0: 0.17553368210792542 (0.21250485380490622) + | > loss_disc_real_1: 0.24188008904457092 (0.23388877428240246) + | > loss_disc_real_2: 0.23900049924850464 (0.2344478600554996) + | > loss_disc_real_3: 0.20081475377082825 (0.23740021304951775) + | > loss_disc_real_4: 0.24353821575641632 (0.24129460089736515) + | > loss_disc_real_5: 0.26134663820266724 (0.24519049376249313) + | > loss_0: 2.7277259826660156 (2.7521318594614663) + | > grad_norm_0: tensor(8.1066, device='cuda:0') (tensor(13.3295, device='cuda:0')) + | > loss_gen: 1.8459954261779785 (1.9162388973765903) + | > loss_kl: 2.5553908348083496 (2.658585124545627) + | > loss_feat: 3.030198574066162 (3.5197479724884033) + | > loss_mel: 20.131576538085938 (21.187366167704266) + | > loss_duration: 1.3429207801818848 (1.3398840294943914) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 28.906082153320312 (30.62182225121392) + | > grad_norm_1: tensor(163.6758, device='cuda:0') (tensor(106.3741, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.7697 (0.7709471649593778) + | > loader_time: 0.0092 (0.008910920884874132) + + + --> TIME: 2024-06-25 07:13:33 -- STEP: 43/591 -- GLOBAL_STEP: 16000 + | > loss_disc: 2.647346019744873 (2.7317422323448706) + | > loss_disc_real_0: 0.21204786002635956 (0.21283040680857593) + | > loss_disc_real_1: 0.19092392921447754 (0.22864904479925022) + | > loss_disc_real_2: 0.24903541803359985 (0.23300117323564928) + | > loss_disc_real_3: 0.23914599418640137 (0.23475540793219277) + | > loss_disc_real_4: 0.24079300463199615 (0.23706151579701623) + | > loss_disc_real_5: 0.2155039757490158 (0.2400831864323727) + | > loss_0: 2.647346019744873 (2.7317422323448706) + | > grad_norm_0: tensor(4.6745, device='cuda:0') (tensor(10.8329, device='cuda:0')) + | > loss_gen: 1.9981085062026978 (1.9004725689111754) + | > loss_kl: 2.644948959350586 (2.6387006016664727) + | > loss_feat: 3.7650325298309326 (3.5005575058072114) + | > loss_mel: 21.43897247314453 (21.190648145453874) + | > loss_duration: 1.3848614692687988 (1.3468945608582608) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.231922149658203 (30.577273524084756) + | > grad_norm_1: tensor(74.9324, device='cuda:0') (tensor(91.0704, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.8079 (0.784850231436796) + | > loader_time: 0.0112 (0.009500780770944994) + + + --> TIME: 2024-06-25 07:13:54 -- STEP: 68/591 -- GLOBAL_STEP: 16025 + | > loss_disc: 2.740792989730835 (2.7288287317051605) + | > loss_disc_real_0: 0.224466934800148 (0.21240698842003064) + | > loss_disc_real_1: 0.25044748187065125 (0.22946822336491415) + | > loss_disc_real_2: 0.246839240193367 (0.23376836803029566) + | > loss_disc_real_3: 0.25374314188957214 (0.23474364596254685) + | > loss_disc_real_4: 0.2363085001707077 (0.2347710553337546) + | > loss_disc_real_5: 0.24807141721248627 (0.23933438080198624) + | > loss_0: 2.740792989730835 (2.7288287317051605) + | > grad_norm_0: tensor(4.0101, device='cuda:0') (tensor(10.4941, device='cuda:0')) + | > loss_gen: 2.0229403972625732 (1.902048524688272) + | > loss_kl: 2.3945088386535645 (2.613073054481955) + | > loss_feat: 3.5268492698669434 (3.506251096725464) + | > loss_mel: 21.743762969970703 (21.211612673366773) + | > loss_duration: 1.3674206733703613 (1.3536076668430777) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.05548095703125 (30.58659309499404) + | > grad_norm_1: tensor(167.3873, device='cuda:0') (tensor(102.3206, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.8284 (0.7953205038519466) + | > loader_time: 0.0111 (0.009824528413660386) + + + --> TIME: 2024-06-25 07:14:15 -- STEP: 93/591 -- GLOBAL_STEP: 16050 + | > loss_disc: 2.745818853378296 (2.73344180660863) + | > loss_disc_real_0: 0.15159450471401215 (0.21227927065344263) + | > loss_disc_real_1: 0.28808990120887756 (0.22933763473905544) + | > loss_disc_real_2: 0.22820177674293518 (0.23522454436107348) + | > loss_disc_real_3: 0.23197467625141144 (0.23558643139818664) + | > loss_disc_real_4: 0.20477408170700073 (0.23510550763658297) + | > loss_disc_real_5: 0.19419899582862854 (0.23926751684117062) + | > loss_0: 2.745818853378296 (2.73344180660863) + | > grad_norm_0: tensor(12.3915, device='cuda:0') (tensor(10.3415, device='cuda:0')) + | > loss_gen: 1.8986271619796753 (1.8977725403283232) + | > loss_kl: 2.450185537338257 (2.602445043543335) + | > loss_feat: 3.6060380935668945 (3.4912016494299776) + | > loss_mel: 21.553495407104492 (21.22928748592254) + | > loss_duration: 1.4041084051132202 (1.3606612195250811) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.912452697753906 (30.581368107949533) + | > grad_norm_1: tensor(196.8668, device='cuda:0') (tensor(117.7058, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.8614 (0.8042700444498369) + | > loader_time: 0.0118 (0.010141526499102195) + + + --> TIME: 2024-06-25 07:14:37 -- STEP: 118/591 -- GLOBAL_STEP: 16075 + | > loss_disc: 2.8866493701934814 (2.7342579708261) + | > loss_disc_real_0: 0.16813315451145172 (0.2122757045268002) + | > loss_disc_real_1: 0.2420421987771988 (0.2290860550888514) + | > loss_disc_real_2: 0.17678025364875793 (0.23484335119946528) + | > loss_disc_real_3: 0.23576368391513824 (0.23592760401257015) + | > loss_disc_real_4: 0.22293461859226227 (0.2349410584922564) + | > loss_disc_real_5: 0.22174082696437836 (0.23913971410464432) + | > loss_0: 2.8866493701934814 (2.7342579708261) + | > grad_norm_0: tensor(9.4663, device='cuda:0') (tensor(10.7103, device='cuda:0')) + | > loss_gen: 2.0191447734832764 (1.8969542677119626) + | > loss_kl: 2.5247559547424316 (2.5818952402826096) + | > loss_feat: 3.5103912353515625 (3.487279273695865) + | > loss_mel: 21.407817840576172 (21.23780863163835) + | > loss_duration: 1.383622646331787 (1.3627589547027978) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.845731735229492 (30.566696474107644) + | > grad_norm_1: tensor(131.9534, device='cuda:0') (tensor(116.4294, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.8287 (0.8125107207540737) + | > loader_time: 0.0115 (0.010373679258055607) + + + --> TIME: 2024-06-25 07:14:58 -- STEP: 143/591 -- GLOBAL_STEP: 16100 + | > loss_disc: 2.681924819946289 (2.7325170657017845) + | > loss_disc_real_0: 0.3012735843658447 (0.21236312509208288) + | > loss_disc_real_1: 0.1934058666229248 (0.22865873981605872) + | > loss_disc_real_2: 0.23387382924556732 (0.2355102907110761) + | > loss_disc_real_3: 0.2067042589187622 (0.2352165958056083) + | > loss_disc_real_4: 0.23098599910736084 (0.23489285672044422) + | > loss_disc_real_5: 0.2222422957420349 (0.23914817361148089) + | > loss_0: 2.681924819946289 (2.7325170657017845) + | > grad_norm_0: tensor(16.1104, device='cuda:0') (tensor(10.2997, device='cuda:0')) + | > loss_gen: 1.9408038854599 (1.8937015383393614) + | > loss_kl: 2.602170944213867 (2.5753513149448226) + | > loss_feat: 3.7872233390808105 (3.4800995913418857) + | > loss_mel: 21.0418758392334 (21.21526564751472) + | > loss_duration: 1.3378427028656006 (1.3626195959277925) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.709917068481445 (30.527037747256404) + | > grad_norm_1: tensor(200.7563, device='cuda:0') (tensor(113.1208, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.848 (0.8186662263803548) + | > loader_time: 0.0128 (0.010604001425362967) + + + --> TIME: 2024-06-25 07:15:20 -- STEP: 168/591 -- GLOBAL_STEP: 16125 + | > loss_disc: 2.7869362831115723 (2.733698527018229) + | > loss_disc_real_0: 0.17636215686798096 (0.21232762107891687) + | > loss_disc_real_1: 0.19939129054546356 (0.2288520659896589) + | > loss_disc_real_2: 0.23492829501628876 (0.23593561049728168) + | > loss_disc_real_3: 0.29085561633110046 (0.2356957568831387) + | > loss_disc_real_4: 0.24603748321533203 (0.23510609886475972) + | > loss_disc_real_5: 0.24167512357234955 (0.2389697508797759) + | > loss_0: 2.7869362831115723 (2.733698527018229) + | > grad_norm_0: tensor(8.7939, device='cuda:0') (tensor(10.3673, device='cuda:0')) + | > loss_gen: 1.9401824474334717 (1.8952881835755848) + | > loss_kl: 2.417468309402466 (2.5678185778004794) + | > loss_feat: 3.206376075744629 (3.479129576966876) + | > loss_mel: 21.153905868530273 (21.20674481846038) + | > loss_duration: 1.3453192710876465 (1.3608031677348273) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.063251495361328 (30.509784414654685) + | > grad_norm_1: tensor(144.8747, device='cuda:0') (tensor(117.4260, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.8618 (0.8245885599227178) + | > loader_time: 0.0132 (0.010812965177354358) + + + --> TIME: 2024-06-25 07:15:42 -- STEP: 193/591 -- GLOBAL_STEP: 16150 + | > loss_disc: 2.746156692504883 (2.733291971868802) + | > loss_disc_real_0: 0.10414881259202957 (0.21165421323300637) + | > loss_disc_real_1: 0.22748525440692902 (0.2291274947051557) + | > loss_disc_real_2: 0.23100286722183228 (0.23604642812143337) + | > loss_disc_real_3: 0.23329196870326996 (0.23558266792890323) + | > loss_disc_real_4: 0.2367262840270996 (0.2348429411649704) + | > loss_disc_real_5: 0.21240989863872528 (0.23898713645848585) + | > loss_0: 2.746156692504883 (2.733291971868802) + | > grad_norm_0: tensor(42.2368, device='cuda:0') (tensor(10., device='cuda:0')) + | > loss_gen: 2.3365087509155273 (1.89685869093386) + | > loss_kl: 2.5477824211120605 (2.566148541751922) + | > loss_feat: 3.691298723220825 (3.4771180029360123) + | > loss_mel: 21.379222869873047 (21.191676283129762) + | > loss_duration: 1.3274424076080322 (1.3594341623968405) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.282257080078125 (30.491235772562767) + | > grad_norm_1: tensor(149.0883, device='cuda:0') (tensor(119.0917, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.8872 (0.8299845697966264) + | > loader_time: 0.0131 (0.011036087194254977) + + + --> TIME: 2024-06-25 07:16:04 -- STEP: 218/591 -- GLOBAL_STEP: 16175 + | > loss_disc: 2.766270160675049 (2.7367909031176785) + | > loss_disc_real_0: 0.188130185008049 (0.21269105292788332) + | > loss_disc_real_1: 0.23693324625492096 (0.2292677507898129) + | > loss_disc_real_2: 0.2428005486726761 (0.2371487655770888) + | > loss_disc_real_3: 0.2524682283401489 (0.23560982470938918) + | > loss_disc_real_4: 0.23190468549728394 (0.23491574482086602) + | > loss_disc_real_5: 0.2243756204843521 (0.23904132535424802) + | > loss_0: 2.766270160675049 (2.7367909031176785) + | > grad_norm_0: tensor(5.8598, device='cuda:0') (tensor(10.2476, device='cuda:0')) + | > loss_gen: 1.8379825353622437 (1.892527744857543) + | > loss_kl: 2.74005126953125 (2.5607277636134307) + | > loss_feat: 3.3528783321380615 (3.465524111319026) + | > loss_mel: 21.106264114379883 (21.187185934924212) + | > loss_duration: 1.3432118892669678 (1.3588203109732457) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.380388259887695 (30.464785952086842) + | > grad_norm_1: tensor(56.5376, device='cuda:0') (tensor(117.0427, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.8907 (0.8349711884052382) + | > loader_time: 0.0148 (0.011335316054317927) + + + --> TIME: 2024-06-25 07:16:27 -- STEP: 243/591 -- GLOBAL_STEP: 16200 + | > loss_disc: 2.735252618789673 (2.736021692370191) + | > loss_disc_real_0: 0.17665939033031464 (0.21273416881698634) + | > loss_disc_real_1: 0.20686233043670654 (0.22886236067171448) + | > loss_disc_real_2: 0.2465125471353531 (0.23768298487359113) + | > loss_disc_real_3: 0.2783249318599701 (0.2358609504169888) + | > loss_disc_real_4: 0.23257049918174744 (0.23488938857498484) + | > loss_disc_real_5: 0.2303743064403534 (0.23888452725155362) + | > loss_0: 2.735252618789673 (2.736021692370191) + | > grad_norm_0: tensor(7.6085, device='cuda:0') (tensor(10.3209, device='cuda:0')) + | > loss_gen: 1.8027920722961426 (1.8945793973075018) + | > loss_kl: 2.573759078979492 (2.5601856885132976) + | > loss_feat: 3.3838400840759277 (3.4686930738849404) + | > loss_mel: 21.004005432128906 (21.190544834843397) + | > loss_duration: 1.339916706085205 (1.3591052995297148) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.104312896728516 (30.47310836226852) + | > grad_norm_1: tensor(75.4910, device='cuda:0') (tensor(116.6248, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.8909 (0.8401208365405047) + | > loader_time: 0.0161 (0.01162156254175759) + + + --> TIME: 2024-06-25 07:16:50 -- STEP: 268/591 -- GLOBAL_STEP: 16225 + | > loss_disc: 2.795971393585205 (2.7379146743176586) + | > loss_disc_real_0: 0.27149486541748047 (0.2132450580708126) + | > loss_disc_real_1: 0.2267897129058838 (0.22919386775413556) + | > loss_disc_real_2: 0.2514743208885193 (0.2375649887679228) + | > loss_disc_real_3: 0.22766955196857452 (0.23536927692258536) + | > loss_disc_real_4: 0.22953402996063232 (0.23475722629410117) + | > loss_disc_real_5: 0.21178369224071503 (0.23899950053709657) + | > loss_0: 2.795971393585205 (2.7379146743176586) + | > grad_norm_0: tensor(9.1332, device='cuda:0') (tensor(10.3707, device='cuda:0')) + | > loss_gen: 2.010047674179077 (1.8930554585670358) + | > loss_kl: 2.517413377761841 (2.557621922955585) + | > loss_feat: 3.426182746887207 (3.4608864241571573) + | > loss_mel: 20.852096557617188 (21.18628027190024) + | > loss_duration: 1.338598608970642 (1.358990566944008) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.144338607788086 (30.456834700570177) + | > grad_norm_1: tensor(126.6232, device='cuda:0') (tensor(115.0484, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.8882 (0.8458655298645816) + | > loader_time: 0.0157 (0.011904075074551709) + + + --> TIME: 2024-06-25 07:17:13 -- STEP: 293/591 -- GLOBAL_STEP: 16250 + | > loss_disc: 2.7817564010620117 (2.7371533144862994) + | > loss_disc_real_0: 0.21638111770153046 (0.213232928524652) + | > loss_disc_real_1: 0.22518929839134216 (0.22905938367998233) + | > loss_disc_real_2: 0.2507208287715912 (0.2378823366079721) + | > loss_disc_real_3: 0.2913416922092438 (0.23559221874528372) + | > loss_disc_real_4: 0.23537467420101166 (0.23443841776546764) + | > loss_disc_real_5: 0.2630278468132019 (0.23886821029943003) + | > loss_0: 2.7817564010620117 (2.7371533144862994) + | > grad_norm_0: tensor(3.8421, device='cuda:0') (tensor(10.1963, device='cuda:0')) + | > loss_gen: 1.8113142251968384 (1.8919762324147664) + | > loss_kl: 2.5136914253234863 (2.5532266027691435) + | > loss_feat: 3.396759271621704 (3.457664204132028) + | > loss_mel: 20.767318725585938 (21.18510498203109) + | > loss_duration: 1.365693211555481 (1.3597263162860278) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.85477638244629 (30.447698391338257) + | > grad_norm_1: tensor(60.1796, device='cuda:0') (tensor(114.2102, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.907 (0.8512960993796082) + | > loader_time: 0.0151 (0.012171113043514942) + + + --> TIME: 2024-06-25 07:17:37 -- STEP: 318/591 -- GLOBAL_STEP: 16275 + | > loss_disc: 2.734117269515991 (2.7358147250781277) + | > loss_disc_real_0: 0.210733562707901 (0.21294387629103362) + | > loss_disc_real_1: 0.23721209168434143 (0.2290918878601782) + | > loss_disc_real_2: 0.2384372055530548 (0.23761649054926146) + | > loss_disc_real_3: 0.24475018680095673 (0.23550187369937417) + | > loss_disc_real_4: 0.24266435205936432 (0.23433952019462045) + | > loss_disc_real_5: 0.26108404994010925 (0.2387481428354791) + | > loss_0: 2.734117269515991 (2.7358147250781277) + | > grad_norm_0: tensor(3.5123, device='cuda:0') (tensor(10.0903, device='cuda:0')) + | > loss_gen: 1.8238402605056763 (1.8925790318153188) + | > loss_kl: 2.3989758491516113 (2.5476794902633584) + | > loss_feat: 3.3255698680877686 (3.4593262012649633) + | > loss_mel: 21.586336135864258 (21.193045322250278) + | > loss_duration: 1.3964731693267822 (1.3607748383246117) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.53119659423828 (30.453404960392405) + | > grad_norm_1: tensor(71.1143, device='cuda:0') (tensor(115.3647, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.9053 (0.8566079821976479) + | > loader_time: 0.0144 (0.012407332846203685) + + + --> TIME: 2024-06-25 07:18:01 -- STEP: 343/591 -- GLOBAL_STEP: 16300 + | > loss_disc: 2.727884531021118 (2.735104665116737) + | > loss_disc_real_0: 0.25569674372673035 (0.21300221397473582) + | > loss_disc_real_1: 0.26428496837615967 (0.2290503449151537) + | > loss_disc_real_2: 0.21685700118541718 (0.23730409684056095) + | > loss_disc_real_3: 0.19644130766391754 (0.23527269860398317) + | > loss_disc_real_4: 0.21336710453033447 (0.23426389437896517) + | > loss_disc_real_5: 0.2478708028793335 (0.23869342602376686) + | > loss_0: 2.727884531021118 (2.735104665116737) + | > grad_norm_0: tensor(9.1291, device='cuda:0') (tensor(10.2282, device='cuda:0')) + | > loss_gen: 2.0550537109375 (1.8922139573027719) + | > loss_kl: 2.5015347003936768 (2.5430065695815474) + | > loss_feat: 3.5590760707855225 (3.4572004229264772) + | > loss_mel: 20.98151969909668 (21.178030859277136) + | > loss_duration: 1.3361648321151733 (1.3618227260800895) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.433347702026367 (30.432274593208692) + | > grad_norm_1: tensor(290.2557, device='cuda:0') (tensor(116.5607, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.9315 (0.8623343584488853) + | > loader_time: 0.0138 (0.012655596691387383) + + + --> TIME: 2024-06-25 07:18:25 -- STEP: 368/591 -- GLOBAL_STEP: 16325 + | > loss_disc: 2.7179853916168213 (2.7337768894174834) + | > loss_disc_real_0: 0.12981154024600983 (0.2126817320999892) + | > loss_disc_real_1: 0.23037035763263702 (0.2289216614368817) + | > loss_disc_real_2: 0.22699826955795288 (0.23711743533773266) + | > loss_disc_real_3: 0.2303602397441864 (0.23545896897659355) + | > loss_disc_real_4: 0.20637696981430054 (0.23424374522722285) + | > loss_disc_real_5: 0.20948919653892517 (0.23852755554506314) + | > loss_0: 2.7179853916168213 (2.7337768894174834) + | > grad_norm_0: tensor(17.7695, device='cuda:0') (tensor(10.1318, device='cuda:0')) + | > loss_gen: 1.7848758697509766 (1.8915773947601733) + | > loss_kl: 2.4299845695495605 (2.5381393205860396) + | > loss_feat: 3.3016443252563477 (3.45307963827382) + | > loss_mel: 20.38624382019043 (21.173908700113717) + | > loss_duration: 1.3689210414886475 (1.3635447414025) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.271669387817383 (30.420249876768693) + | > grad_norm_1: tensor(229.9881, device='cuda:0') (tensor(119.9697, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.9604 (0.86788909331612) + | > loader_time: 0.0185 (0.012913172011790064) + + + --> TIME: 2024-06-25 07:18:49 -- STEP: 393/591 -- GLOBAL_STEP: 16350 + | > loss_disc: 2.814005136489868 (2.734191418907419) + | > loss_disc_real_0: 0.34777629375457764 (0.21262789106247687) + | > loss_disc_real_1: 0.2294069230556488 (0.22900621581623573) + | > loss_disc_real_2: 0.23961135745048523 (0.23708398120700555) + | > loss_disc_real_3: 0.26991432905197144 (0.23566159928269664) + | > loss_disc_real_4: 0.25568681955337524 (0.23438673957461922) + | > loss_disc_real_5: 0.24561312794685364 (0.23865026899118155) + | > loss_0: 2.814005136489868 (2.734191418907419) + | > grad_norm_0: tensor(24.7034, device='cuda:0') (tensor(10.1051, device='cuda:0')) + | > loss_gen: 1.7187740802764893 (1.8913559215972746) + | > loss_kl: 2.4743804931640625 (2.5344147712523113) + | > loss_feat: 3.1994638442993164 (3.4468153032637736) + | > loss_mel: 21.939422607421875 (21.16650150871763) + | > loss_duration: 1.3714734315872192 (1.3643819888432807) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.703514099121094 (30.40346956131719) + | > grad_norm_1: tensor(57.5205, device='cuda:0') (tensor(118.6759, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 0.9611 (0.8739625904092961) + | > loader_time: 0.0169 (0.013172457843033104) + + + --> TIME: 2024-06-25 07:19:15 -- STEP: 418/591 -- GLOBAL_STEP: 16375 + | > loss_disc: 2.7487478256225586 (2.7341799211273936) + | > loss_disc_real_0: 0.15727630257606506 (0.21235341534161112) + | > loss_disc_real_1: 0.23166462779045105 (0.22889015941243424) + | > loss_disc_real_2: 0.23134030401706696 (0.23706879552328985) + | > loss_disc_real_3: 0.22128112614154816 (0.23569572638524205) + | > loss_disc_real_4: 0.20689581334590912 (0.2341679740679321) + | > loss_disc_real_5: 0.22958824038505554 (0.23854844653577895) + | > loss_0: 2.7487478256225586 (2.7341799211273936) + | > grad_norm_0: tensor(10.5185, device='cuda:0') (tensor(10.1065, device='cuda:0')) + | > loss_gen: 1.9808913469314575 (1.8903976031467675) + | > loss_kl: 2.527618408203125 (2.5308250827652436) + | > loss_feat: 3.6417951583862305 (3.4415023560729323) + | > loss_mel: 20.97917938232422 (21.162321998742218) + | > loss_duration: 1.3044915199279785 (1.3651658684443044) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.433977127075195 (30.390212976191034) + | > grad_norm_1: tensor(40.9111, device='cuda:0') (tensor(118.0080, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 1.0026 (0.8811690590598369) + | > loader_time: 0.019 (0.013457611987465302) + + + --> TIME: 2024-06-25 07:19:41 -- STEP: 443/591 -- GLOBAL_STEP: 16400 + | > loss_disc: 2.733111619949341 (2.7339658973986634) + | > loss_disc_real_0: 0.18533368408679962 (0.21227328864432887) + | > loss_disc_real_1: 0.24556826055049896 (0.2289412393997808) + | > loss_disc_real_2: 0.2305716872215271 (0.2370001229018175) + | > loss_disc_real_3: 0.16818542778491974 (0.23569343062475087) + | > loss_disc_real_4: 0.22899451851844788 (0.23423817340057537) + | > loss_disc_real_5: 0.2478129267692566 (0.23852697919642146) + | > loss_0: 2.733111619949341 (2.7339658973986634) + | > grad_norm_0: tensor(10.0489, device='cuda:0') (tensor(10.1529, device='cuda:0')) + | > loss_gen: 1.9896844625473022 (1.8900420687268604) + | > loss_kl: 2.456061601638794 (2.5271081466976213) + | > loss_feat: 3.3078112602233887 (3.4354927658227443) + | > loss_mel: 21.218975067138672 (21.15413385481652) + | > loss_duration: 1.4080088138580322 (1.3663819091852833) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.380542755126953 (30.37315882301761) + | > grad_norm_1: tensor(75.8138, device='cuda:0') (tensor(117.4658, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 1.0459 (0.8880906723961068) + | > loader_time: 0.0159 (0.01374779221166606) + + + --> TIME: 2024-06-25 07:20:07 -- STEP: 468/591 -- GLOBAL_STEP: 16425 + | > loss_disc: 2.830660343170166 (2.7356618617334947) + | > loss_disc_real_0: 0.08118875324726105 (0.21228535685083297) + | > loss_disc_real_1: 0.227784663438797 (0.22918412528741053) + | > loss_disc_real_2: 0.21681804955005646 (0.23709283565354144) + | > loss_disc_real_3: 0.2518479526042938 (0.2360769061642325) + | > loss_disc_real_4: 0.2319173663854599 (0.23429784907871842) + | > loss_disc_real_5: 0.2331300675868988 (0.23849749510996363) + | > loss_0: 2.830660343170166 (2.7356618617334947) + | > grad_norm_0: tensor(33.9053, device='cuda:0') (tensor(10.2942, device='cuda:0')) + | > loss_gen: 2.0265867710113525 (1.8892488568766503) + | > loss_kl: 2.481060743331909 (2.5242993638046807) + | > loss_feat: 3.2927815914154053 (3.4256072487586584) + | > loss_mel: 21.203655242919922 (21.137223166278293) + | > loss_duration: 1.3905166387557983 (1.36743312411838) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.39459991455078 (30.34381180950719) + | > grad_norm_1: tensor(112.3234, device='cuda:0') (tensor(115.4380, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 1.0279 (0.895532231555026) + | > loader_time: 0.0194 (0.014013130950112624) + + + --> TIME: 2024-06-25 07:20:34 -- STEP: 493/591 -- GLOBAL_STEP: 16450 + | > loss_disc: 2.761338949203491 (2.735773771336307) + | > loss_disc_real_0: 0.2787650227546692 (0.21263304460060767) + | > loss_disc_real_1: 0.21308204531669617 (0.2293300794297735) + | > loss_disc_real_2: 0.24257157742977142 (0.2370624445514795) + | > loss_disc_real_3: 0.25013697147369385 (0.2360528223896607) + | > loss_disc_real_4: 0.229209765791893 (0.23422516323248224) + | > loss_disc_real_5: 0.2465410679578781 (0.23855867882166626) + | > loss_0: 2.761338949203491 (2.735773771336307) + | > grad_norm_0: tensor(11.5299, device='cuda:0') (tensor(10.3619, device='cuda:0')) + | > loss_gen: 1.8342783451080322 (1.8890327135399438) + | > loss_kl: 2.4278550148010254 (2.5211105985041824) + | > loss_feat: 3.266589879989624 (3.4215108869525532) + | > loss_mel: 21.12129020690918 (21.123723991502377) + | > loss_duration: 1.3777003288269043 (1.3687905375420673) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.027713775634766 (30.324168775918277) + | > grad_norm_1: tensor(172.2576, device='cuda:0') (tensor(114.4999, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 1.0204 (0.9034760278813981) + | > loader_time: 0.0205 (0.014316734387473206) + + + --> TIME: 2024-06-25 07:21:02 -- STEP: 518/591 -- GLOBAL_STEP: 16475 + | > loss_disc: 2.704848289489746 (2.735374133099001) + | > loss_disc_real_0: 0.14764872193336487 (0.21231390010838802) + | > loss_disc_real_1: 0.24758702516555786 (0.22929519248054755) + | > loss_disc_real_2: 0.21779324114322662 (0.23719433338370563) + | > loss_disc_real_3: 0.2353314459323883 (0.23594918522020103) + | > loss_disc_real_4: 0.2380855828523636 (0.23420457472902467) + | > loss_disc_real_5: 0.20543763041496277 (0.23850898804236564) + | > loss_0: 2.704848289489746 (2.735374133099001) + | > grad_norm_0: tensor(13.4146, device='cuda:0') (tensor(10.2900, device='cuda:0')) + | > loss_gen: 2.0949230194091797 (1.8885845003441033) + | > loss_kl: 2.5190811157226562 (2.518262397368444) + | > loss_feat: 3.4025299549102783 (3.4146692743632783) + | > loss_mel: 21.03752326965332 (21.1097136979858) + | > loss_duration: 1.3740036487579346 (1.370330575810435) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.42806053161621 (30.30156049028787) + | > grad_norm_1: tensor(53.6452, device='cuda:0') (tensor(115.1143, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 1.1358 (0.9129393050109099) + | > loader_time: 0.0193 (0.0146138470145266) + + + --> TIME: 2024-06-25 07:21:31 -- STEP: 543/591 -- GLOBAL_STEP: 16500 + | > loss_disc: 2.767648696899414 (2.7359721098596013) + | > loss_disc_real_0: 0.22354725003242493 (0.2123406492791123) + | > loss_disc_real_1: 0.24766874313354492 (0.2294946877392177) + | > loss_disc_real_2: 0.22118064761161804 (0.23710842814783584) + | > loss_disc_real_3: 0.2360026091337204 (0.23604920794621356) + | > loss_disc_real_4: 0.23202966153621674 (0.23428310533474386) + | > loss_disc_real_5: 0.2170374095439911 (0.23861104754892282) + | > loss_0: 2.767648696899414 (2.7359721098596013) + | > grad_norm_0: tensor(6.6910, device='cuda:0') (tensor(10.2028, device='cuda:0')) + | > loss_gen: 1.7786083221435547 (1.8872724119470923) + | > loss_kl: 2.5369246006011963 (2.5153763483022917) + | > loss_feat: 2.9923136234283447 (3.4070090237682478) + | > loss_mel: 20.331069946289062 (21.10266862521515) + | > loss_duration: 1.4123209714889526 (1.3717784218586162) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.051237106323242 (30.284104854801523) + | > grad_norm_1: tensor(53.6063, device='cuda:0') (tensor(114.7626, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 1.1766 (0.9231454781625372) + | > loader_time: 0.0228 (0.014952297368760926) + + + --> TIME: 2024-06-25 07:22:02 -- STEP: 568/591 -- GLOBAL_STEP: 16525 + | > loss_disc: 2.777033805847168 (2.7361120548046824) + | > loss_disc_real_0: 0.21803070604801178 (0.21247394825003937) + | > loss_disc_real_1: 0.2541770040988922 (0.22949304174579366) + | > loss_disc_real_2: 0.27099859714508057 (0.23709993263785267) + | > loss_disc_real_3: 0.23237450420856476 (0.23602851897373167) + | > loss_disc_real_4: 0.22111794352531433 (0.23419905280773068) + | > loss_disc_real_5: 0.26021912693977356 (0.23860688840734287) + | > loss_0: 2.777033805847168 (2.7361120548046824) + | > grad_norm_0: tensor(5.4827, device='cuda:0') (tensor(10.2217, device='cuda:0')) + | > loss_gen: 1.795544147491455 (1.8869388989159759) + | > loss_kl: 2.3373305797576904 (2.5124987842331494) + | > loss_feat: 2.959892988204956 (3.40110615861248) + | > loss_mel: 20.657419204711914 (21.091892524504328) + | > loss_duration: 1.393868088722229 (1.3733013098088775) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.144054412841797 (30.265737701469742) + | > grad_norm_1: tensor(80.3105, device='cuda:0') (tensor(116.1481, device='cuda:0')) + | > current_lr_0: 0.00019932609573327815 + | > current_lr_1: 0.00019932609573327815 + | > step_time: 1.1912 (0.9352327879046051) + | > loader_time: 0.025 (0.015309023605266087) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036790523123233876 (+1.4203660031594442e-07) + | > avg_loss_disc: 2.757897950233297 (+0.02334634801174662) + | > avg_loss_disc_real_0: 0.17708961935119427 (-0.16713532852999707) + | > avg_loss_disc_real_1: 0.2707041429078325 (+0.027338200744162183) + | > avg_loss_disc_real_2: 0.29143908366243887 (+0.026306385372547314) + | > avg_loss_disc_real_3: 0.2902264528452081 (+0.06137618707849615) + | > avg_loss_disc_real_4: 0.2529054937210488 (+0.019439798720339435) + | > avg_loss_disc_real_5: 0.2570066861015685 (+0.018959090430685832) + | > avg_loss_0: 2.757897950233297 (+0.02334634801174662) + | > avg_loss_gen: 2.00603537102963 (-0.09516751766205012) + | > avg_loss_kl: 2.5745317022851175 (-0.01091024216185188) + | > avg_loss_feat: 3.3341794115431767 (+0.05505807856295997) + | > avg_loss_mel: 21.089648023564756 (+0.16913450525161267) + | > avg_loss_duration: 1.3927187386979447 (+0.005502305132277341) + | > avg_loss_1: 30.397113069574882 (+0.12361672584046701) + + + > EPOCH: 29/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 07:22:39)  + + --> TIME: 2024-06-25 07:22:42 -- STEP: 2/591 -- GLOBAL_STEP: 16550 + | > loss_disc: 2.7513175010681152 (2.786540985107422) + | > loss_disc_real_0: 0.20949876308441162 (0.25342370569705963) + | > loss_disc_real_1: 0.18878594040870667 (0.21049124747514725) + | > loss_disc_real_2: 0.2250777930021286 (0.23793425410985947) + | > loss_disc_real_3: 0.22230647504329681 (0.2494717761874199) + | > loss_disc_real_4: 0.240225151181221 (0.24496982991695404) + | > loss_disc_real_5: 0.26570576429367065 (0.254342682659626) + | > loss_0: 2.7513175010681152 (2.786540985107422) + | > grad_norm_0: tensor(4.7529, device='cuda:0') (tensor(8.8642, device='cuda:0')) + | > loss_gen: 1.8890511989593506 (1.840647578239441) + | > loss_kl: 2.5695388317108154 (2.750886917114258) + | > loss_feat: 3.567809581756592 (3.628737688064575) + | > loss_mel: 22.06096839904785 (21.9343843460083) + | > loss_duration: 1.3706398010253906 (1.372513771057129) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.4580078125 (31.527170181274414) + | > grad_norm_1: tensor(129.3990, device='cuda:0') (tensor(198.3413, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.7587 (0.7573848962783813) + | > loader_time: 0.0081 (0.007771968841552734) + + + --> TIME: 2024-06-25 07:23:02 -- STEP: 27/591 -- GLOBAL_STEP: 16575 + | > loss_disc: 2.7354466915130615 (2.7581046245716236) + | > loss_disc_real_0: 0.2872793674468994 (0.21733841024063252) + | > loss_disc_real_1: 0.21621300280094147 (0.23507543680844484) + | > loss_disc_real_2: 0.21836987137794495 (0.23644676859732028) + | > loss_disc_real_3: 0.2553003132343292 (0.23826038506295946) + | > loss_disc_real_4: 0.2346574366092682 (0.23753073204446723) + | > loss_disc_real_5: 0.2294723391532898 (0.23972116576300728) + | > loss_0: 2.7354466915130615 (2.7581046245716236) + | > grad_norm_0: tensor(14.5881, device='cuda:0') (tensor(9.4806, device='cuda:0')) + | > loss_gen: 1.7879208326339722 (1.9034658405515883) + | > loss_kl: 2.48431658744812 (2.6428443855709505) + | > loss_feat: 3.4356939792633057 (3.5485276646084256) + | > loss_mel: 21.93488311767578 (21.552525696931063) + | > loss_duration: 1.3341948986053467 (1.3358131338048866) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.977006912231445 (30.983176902488427) + | > grad_norm_1: tensor(168.4094, device='cuda:0') (tensor(186.0946, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.7937 (0.7889786914542869) + | > loader_time: 0.009 (0.00911908679538303) + + + --> TIME: 2024-06-25 07:23:22 -- STEP: 52/591 -- GLOBAL_STEP: 16600 + | > loss_disc: 2.7082180976867676 (2.7418266351406393) + | > loss_disc_real_0: 0.2348739057779312 (0.21420775841061884) + | > loss_disc_real_1: 0.2160004824399948 (0.22932046164686865) + | > loss_disc_real_2: 0.25120487809181213 (0.23630293143483308) + | > loss_disc_real_3: 0.24818794429302216 (0.2357198022879087) + | > loss_disc_real_4: 0.23808617889881134 (0.23602849789536917) + | > loss_disc_real_5: 0.2600856125354767 (0.23996551564106575) + | > loss_0: 2.7082180976867676 (2.7418266351406393) + | > grad_norm_0: tensor(4.5257, device='cuda:0') (tensor(10.3987, device='cuda:0')) + | > loss_gen: 1.9747666120529175 (1.9023869794148665) + | > loss_kl: 2.4305648803710938 (2.6136090801312384) + | > loss_feat: 3.6386709213256836 (3.5407322461788473) + | > loss_mel: 21.04098892211914 (21.36865344414344) + | > loss_duration: 1.3453543186187744 (1.347457149854073) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.43034553527832 (30.772838996006893) + | > grad_norm_1: tensor(66.8766, device='cuda:0') (tensor(162.8694, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8249 (0.7953117214716398) + | > loader_time: 0.0094 (0.009467753080221322) + + + --> TIME: 2024-06-25 07:23:43 -- STEP: 77/591 -- GLOBAL_STEP: 16625 + | > loss_disc: 2.664423942565918 (2.736402976048457) + | > loss_disc_real_0: 0.13439887762069702 (0.2132693159502822) + | > loss_disc_real_1: 0.23811601102352142 (0.228664774012256) + | > loss_disc_real_2: 0.2060088962316513 (0.23542352349727186) + | > loss_disc_real_3: 0.226840540766716 (0.23488387213898942) + | > loss_disc_real_4: 0.2069651335477829 (0.2352687267514018) + | > loss_disc_real_5: 0.19926781952381134 (0.24016281594703723) + | > loss_0: 2.664423942565918 (2.736402976048457) + | > grad_norm_0: tensor(16.8977, device='cuda:0') (tensor(10.1653, device='cuda:0')) + | > loss_gen: 1.9716168642044067 (1.8957321055523761) + | > loss_kl: 2.5351767539978027 (2.5880760682093644) + | > loss_feat: 3.7038302421569824 (3.522448849368405) + | > loss_mel: 21.384634017944336 (21.32276203105976) + | > loss_duration: 1.3979734182357788 (1.3534440436920563) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.99323272705078 (30.682463212446734) + | > grad_norm_1: tensor(202.9234, device='cuda:0') (tensor(140.9587, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8316 (0.8036313273689963) + | > loader_time: 0.0107 (0.009847157961362367) + + + --> TIME: 2024-06-25 07:24:04 -- STEP: 102/591 -- GLOBAL_STEP: 16650 + | > loss_disc: 2.6688344478607178 (2.737443853827083) + | > loss_disc_real_0: 0.13921043276786804 (0.21434136611573837) + | > loss_disc_real_1: 0.2036980390548706 (0.2285440735080663) + | > loss_disc_real_2: 0.22149598598480225 (0.2362926375924372) + | > loss_disc_real_3: 0.24856586754322052 (0.23599505146928862) + | > loss_disc_real_4: 0.22633183002471924 (0.23457101191960128) + | > loss_disc_real_5: 0.24018296599388123 (0.23932367957690182) + | > loss_0: 2.6688344478607178 (2.737443853827083) + | > grad_norm_0: tensor(14.2826, device='cuda:0') (tensor(10.5550, device='cuda:0')) + | > loss_gen: 1.8218016624450684 (1.8935610640282725) + | > loss_kl: 2.4592807292938232 (2.5761303387436225) + | > loss_feat: 3.495698928833008 (3.5202203287797817) + | > loss_mel: 21.040807723999023 (21.311920128616627) + | > loss_duration: 1.387413501739502 (1.3603066650091433) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.20500373840332 (30.662138695810356) + | > grad_norm_1: tensor(120.0135, device='cuda:0') (tensor(144.8095, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8261 (0.810590059149499) + | > loader_time: 0.0108 (0.01021866938647102) + + + --> TIME: 2024-06-25 07:24:26 -- STEP: 127/591 -- GLOBAL_STEP: 16675 + | > loss_disc: 2.751835823059082 (2.737734612517469) + | > loss_disc_real_0: 0.22745470702648163 (0.21457389360807072) + | > loss_disc_real_1: 0.22713717818260193 (0.22836272089969456) + | > loss_disc_real_2: 0.2791120111942291 (0.23654315767325754) + | > loss_disc_real_3: 0.24719388782978058 (0.23588611326349063) + | > loss_disc_real_4: 0.25043919682502747 (0.23506889275209172) + | > loss_disc_real_5: 0.2400715947151184 (0.23909333324807835) + | > loss_0: 2.751835823059082 (2.737734612517469) + | > grad_norm_0: tensor(3.8724, device='cuda:0') (tensor(10.5110, device='cuda:0')) + | > loss_gen: 1.7913705110549927 (1.8956155270103394) + | > loss_kl: 2.548344135284424 (2.5613120278035573) + | > loss_feat: 3.4484827518463135 (3.517282810736829) + | > loss_mel: 20.824310302734375 (21.29440061313899) + | > loss_duration: 1.3610544204711914 (1.3618944094875667) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.973560333251953 (30.6305054792269) + | > grad_norm_1: tensor(247.7045, device='cuda:0') (tensor(147.3834, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8571 (0.8171902566444217) + | > loader_time: 0.0127 (0.010464189559455932) + + + --> TIME: 2024-06-25 07:24:48 -- STEP: 152/591 -- GLOBAL_STEP: 16700 + | > loss_disc: 2.73984956741333 (2.7369368092009885) + | > loss_disc_real_0: 0.1984972357749939 (0.21371211802684942) + | > loss_disc_real_1: 0.15965406596660614 (0.22852389877171894) + | > loss_disc_real_2: 0.22787252068519592 (0.23685732287795921) + | > loss_disc_real_3: 0.2115342766046524 (0.23532573290561376) + | > loss_disc_real_4: 0.2500108778476715 (0.23451149267585655) + | > loss_disc_real_5: 0.2647433578968048 (0.23930968089323296) + | > loss_0: 2.73984956741333 (2.7369368092009885) + | > grad_norm_0: tensor(7.0726, device='cuda:0') (tensor(10.4632, device='cuda:0')) + | > loss_gen: 2.0053350925445557 (1.895315610264477) + | > loss_kl: 2.624750852584839 (2.5524004729170557) + | > loss_feat: 3.533792734146118 (3.5128798735769178) + | > loss_mel: 20.950132369995117 (21.263718103107653) + | > loss_duration: 1.342536449432373 (1.3607742425642513) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.4565486907959 (30.585088365956356) + | > grad_norm_1: tensor(59.5884, device='cuda:0') (tensor(145.2420, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8571 (0.8229699291680987) + | > loader_time: 0.0111 (0.01070541300271687) + + + --> TIME: 2024-06-25 07:25:10 -- STEP: 177/591 -- GLOBAL_STEP: 16725 + | > loss_disc: 2.6345183849334717 (2.7367673714955627) + | > loss_disc_real_0: 0.21013571321964264 (0.21388429011833865) + | > loss_disc_real_1: 0.19772601127624512 (0.2285396348958635) + | > loss_disc_real_2: 0.22753502428531647 (0.2368692557858882) + | > loss_disc_real_3: 0.1994543820619583 (0.2355292879760602) + | > loss_disc_real_4: 0.21806101500988007 (0.23398045067396542) + | > loss_disc_real_5: 0.23563523590564728 (0.23905755394259415) + | > loss_0: 2.6345183849334717 (2.7367673714955627) + | > grad_norm_0: tensor(3.4035, device='cuda:0') (tensor(10.4073, device='cuda:0')) + | > loss_gen: 2.1001908779144287 (1.8940099444093004) + | > loss_kl: 2.384753465652466 (2.549337272590164) + | > loss_feat: 3.715867757797241 (3.508069571802172) + | > loss_mel: 20.957042694091797 (21.239569604733564) + | > loss_duration: 1.363612413406372 (1.3589028699249868) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.521467208862305 (30.54988931127861) + | > grad_norm_1: tensor(40.1504, device='cuda:0') (tensor(141.0018, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8802 (0.8291961958179366) + | > loader_time: 0.0126 (0.010974198411413508) + + + --> TIME: 2024-06-25 07:25:32 -- STEP: 202/591 -- GLOBAL_STEP: 16750 + | > loss_disc: 2.713801860809326 (2.7371452185186986) + | > loss_disc_real_0: 0.2595179080963135 (0.21429427606191961) + | > loss_disc_real_1: 0.23232696950435638 (0.2285162725365988) + | > loss_disc_real_2: 0.2123560905456543 (0.2367540897414236) + | > loss_disc_real_3: 0.25219133496284485 (0.23580918394693054) + | > loss_disc_real_4: 0.23652838170528412 (0.23430218672988437) + | > loss_disc_real_5: 0.2623278498649597 (0.23919935074478094) + | > loss_0: 2.713801860809326 (2.7371452185186986) + | > grad_norm_0: tensor(8.4078, device='cuda:0') (tensor(10.8364, device='cuda:0')) + | > loss_gen: 1.7676315307617188 (1.8927414936594444) + | > loss_kl: 2.408926486968994 (2.54650118209348) + | > loss_feat: 3.5343756675720215 (3.5037245478960553) + | > loss_mel: 21.03267478942871 (21.22314524886631) + | > loss_duration: 1.3103387355804443 (1.3579504560716076) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.05394744873047 (30.524062978159083) + | > grad_norm_1: tensor(43.2809, device='cuda:0') (tensor(140.7259, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8873 (0.8343378222814881) + | > loader_time: 0.0145 (0.011225585890288402) + + + --> TIME: 2024-06-25 07:25:54 -- STEP: 227/591 -- GLOBAL_STEP: 16775 + | > loss_disc: 2.6883633136749268 (2.7342575415640655) + | > loss_disc_real_0: 0.18912291526794434 (0.21354390769409184) + | > loss_disc_real_1: 0.2304403930902481 (0.22818586381258943) + | > loss_disc_real_2: 0.2478192299604416 (0.23687881191921653) + | > loss_disc_real_3: 0.24531827867031097 (0.2353617944118735) + | > loss_disc_real_4: 0.2390383630990982 (0.23398785929847918) + | > loss_disc_real_5: 0.23846982419490814 (0.2389920588238124) + | > loss_0: 2.6883633136749268 (2.7342575415640655) + | > grad_norm_0: tensor(5.7384, device='cuda:0') (tensor(10.6696, device='cuda:0')) + | > loss_gen: 1.865828037261963 (1.8930120977536171) + | > loss_kl: 2.553617000579834 (2.5427832939551274) + | > loss_feat: 3.690537929534912 (3.5054037014293256) + | > loss_mel: 20.678754806518555 (21.18952438589759) + | > loss_duration: 1.3700076341629028 (1.35710432823534) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.15874481201172 (30.48782787238974) + | > grad_norm_1: tensor(152.9953, device='cuda:0') (tensor(135.4750, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8703 (0.8391574143313101) + | > loader_time: 0.0153 (0.011526313647299613) + + + --> TIME: 2024-06-25 07:26:17 -- STEP: 252/591 -- GLOBAL_STEP: 16800 + | > loss_disc: 2.706174373626709 (2.731841000299603) + | > loss_disc_real_0: 0.2140730917453766 (0.21320442207867188) + | > loss_disc_real_1: 0.275219202041626 (0.22803550869936035) + | > loss_disc_real_2: 0.2528950273990631 (0.23662784340835752) + | > loss_disc_real_3: 0.1813231259584427 (0.23516339891486698) + | > loss_disc_real_4: 0.2497621327638626 (0.2340901835806786) + | > loss_disc_real_5: 0.24985246360301971 (0.2390502044369304) + | > loss_0: 2.706174373626709 (2.731841000299603) + | > grad_norm_0: tensor(6.9658, device='cuda:0') (tensor(10.3586, device='cuda:0')) + | > loss_gen: 1.8389651775360107 (1.894830094443427) + | > loss_kl: 2.5115413665771484 (2.542533656907461) + | > loss_feat: 3.3830995559692383 (3.5097928236401277) + | > loss_mel: 20.791664123535156 (21.180300243317127) + | > loss_duration: 1.3413978815078735 (1.3576171081217503) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.866668701171875 (30.48507399029202) + | > grad_norm_1: tensor(49.1391, device='cuda:0') (tensor(133.0842, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.8862 (0.844056521143232) + | > loader_time: 0.0159 (0.011845982263958645) + + + --> TIME: 2024-06-25 07:26:40 -- STEP: 277/591 -- GLOBAL_STEP: 16825 + | > loss_disc: 2.681492567062378 (2.7331531624501344) + | > loss_disc_real_0: 0.17531651258468628 (0.2133876385725362) + | > loss_disc_real_1: 0.19430145621299744 (0.22833190514077348) + | > loss_disc_real_2: 0.22085727751255035 (0.23649492859840393) + | > loss_disc_real_3: 0.25044161081314087 (0.23507288388827216) + | > loss_disc_real_4: 0.23526404798030853 (0.23407620724142675) + | > loss_disc_real_5: 0.22855722904205322 (0.23899296232724448) + | > loss_0: 2.681492567062378 (2.7331531624501344) + | > grad_norm_0: tensor(7.4412, device='cuda:0') (tensor(10.5787, device='cuda:0')) + | > loss_gen: 1.8856598138809204 (1.8935267327064211) + | > loss_kl: 2.5416150093078613 (2.539585471583619) + | > loss_feat: 3.536825656890869 (3.5025245093266455) + | > loss_mel: 20.712932586669922 (21.167981977497192) + | > loss_duration: 1.356994867324829 (1.357548606955187) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.034029006958008 (30.46116735272459) + | > grad_norm_1: tensor(90.9026, device='cuda:0') (tensor(134.2624, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.9139 (0.8491367054329884) + | > loader_time: 0.0155 (0.012159191744422227) + + + --> TIME: 2024-06-25 07:27:03 -- STEP: 302/591 -- GLOBAL_STEP: 16850 + | > loss_disc: 2.5990970134735107 (2.7333755445796113) + | > loss_disc_real_0: 0.16069699823856354 (0.21341152526191529) + | > loss_disc_real_1: 0.23724354803562164 (0.22809741591775654) + | > loss_disc_real_2: 0.21941591799259186 (0.23648949376992043) + | > loss_disc_real_3: 0.25067800283432007 (0.23510788346560585) + | > loss_disc_real_4: 0.2084898203611374 (0.23415641851772537) + | > loss_disc_real_5: 0.24311797320842743 (0.23933994429593056) + | > loss_0: 2.5990970134735107 (2.7333755445796113) + | > grad_norm_0: tensor(6.6149, device='cuda:0') (tensor(10.4313, device='cuda:0')) + | > loss_gen: 1.9279193878173828 (1.8917741964984414) + | > loss_kl: 2.5302326679229736 (2.5380268373236765) + | > loss_feat: 3.655033588409424 (3.497154461626975) + | > loss_mel: 21.433162689208984 (21.167517889414395) + | > loss_duration: 1.346344232559204 (1.3580469058838898) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.89269256591797 (30.452520319957607) + | > grad_norm_1: tensor(57.5139, device='cuda:0') (tensor(133.5408, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.9455 (0.8543913703880561) + | > loader_time: 0.0167 (0.012401613178632117) + + + --> TIME: 2024-06-25 07:27:27 -- STEP: 327/591 -- GLOBAL_STEP: 16875 + | > loss_disc: 2.801712989807129 (2.733301166365268) + | > loss_disc_real_0: 0.246036559343338 (0.2135576051901969) + | > loss_disc_real_1: 0.2122984528541565 (0.22822814947421396) + | > loss_disc_real_2: 0.25391456484794617 (0.23628859139910532) + | > loss_disc_real_3: 0.26566046476364136 (0.23529453698648226) + | > loss_disc_real_4: 0.25900882482528687 (0.23414874555320914) + | > loss_disc_real_5: 0.2527969777584076 (0.239150556265761) + | > loss_0: 2.801712989807129 (2.733301166365268) + | > grad_norm_0: tensor(6.0029, device='cuda:0') (tensor(10.4974, device='cuda:0')) + | > loss_gen: 1.7944928407669067 (1.8914832744394239) + | > loss_kl: 2.4840126037597656 (2.5351562113572337) + | > loss_feat: 3.3745174407958984 (3.4941408240467036) + | > loss_mel: 20.921367645263672 (21.158764381292027) + | > loss_duration: 1.3822128772735596 (1.3590331391092467) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.95660400390625 (30.438577867793747) + | > grad_norm_1: tensor(148.6867, device='cuda:0') (tensor(130.7398, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.9237 (0.8595497732133309) + | > loader_time: 0.0161 (0.012647193506223346) + + + --> TIME: 2024-06-25 07:27:51 -- STEP: 352/591 -- GLOBAL_STEP: 16900 + | > loss_disc: 2.7134079933166504 (2.733155713162635) + | > loss_disc_real_0: 0.2167252004146576 (0.21365750676274978) + | > loss_disc_real_1: 0.20462453365325928 (0.22809851605614478) + | > loss_disc_real_2: 0.26736414432525635 (0.23617145533419467) + | > loss_disc_real_3: 0.2289877086877823 (0.23519246610389513) + | > loss_disc_real_4: 0.2204083800315857 (0.2341552586718039) + | > loss_disc_real_5: 0.21378609538078308 (0.23918512641367587) + | > loss_0: 2.7134079933166504 (2.733155713162635) + | > grad_norm_0: tensor(3.3730, device='cuda:0') (tensor(10.6607, device='cuda:0')) + | > loss_gen: 1.9308518171310425 (1.8913708065043797) + | > loss_kl: 2.5024168491363525 (2.532485604286194) + | > loss_feat: 3.3012404441833496 (3.4909358370033177) + | > loss_mel: 21.287281036376953 (21.160787441513747) + | > loss_duration: 1.39252507686615 (1.360397834330797) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.41431427001953 (30.435977572744545) + | > grad_norm_1: tensor(113.5973, device='cuda:0') (tensor(133.4987, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.9331 (0.8652273883873765) + | > loader_time: 0.0181 (0.012884104116396471) + + + --> TIME: 2024-06-25 07:28:15 -- STEP: 377/591 -- GLOBAL_STEP: 16925 + | > loss_disc: 2.6673197746276855 (2.7324023600914713) + | > loss_disc_real_0: 0.2230663001537323 (0.21363897198707418) + | > loss_disc_real_1: 0.22272376716136932 (0.2281770435426216) + | > loss_disc_real_2: 0.22253744304180145 (0.23597452356739448) + | > loss_disc_real_3: 0.23544497787952423 (0.23525800972939803) + | > loss_disc_real_4: 0.22934843599796295 (0.23396364945315556) + | > loss_disc_real_5: 0.22485288977622986 (0.23911224475589923) + | > loss_0: 2.6673197746276855 (2.7324023600914713) + | > grad_norm_0: tensor(3.8943, device='cuda:0') (tensor(10.6780, device='cuda:0')) + | > loss_gen: 1.8579400777816772 (1.8916974504050905) + | > loss_kl: 2.4384191036224365 (2.528951434304924) + | > loss_feat: 3.6471614837646484 (3.4884594960301243) + | > loss_mel: 20.566957473754883 (21.15064112008091) + | > loss_duration: 1.381108283996582 (1.362140522079063) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.891586303710938 (30.421890071595694) + | > grad_norm_1: tensor(87.0350, device='cuda:0') (tensor(131.7358, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 1.0428 (0.8709826115271772) + | > loader_time: 0.0185 (0.013169220967381323) + + + --> TIME: 2024-06-25 07:28:40 -- STEP: 402/591 -- GLOBAL_STEP: 16950 + | > loss_disc: 2.9786577224731445 (2.7335864252118887) + | > loss_disc_real_0: 0.20085349678993225 (0.2138130595138417) + | > loss_disc_real_1: 0.27917516231536865 (0.2285236363757902) + | > loss_disc_real_2: 0.21792861819267273 (0.23599102511186504) + | > loss_disc_real_3: 0.2355414181947708 (0.23519169439130755) + | > loss_disc_real_4: 0.21349549293518066 (0.23396433611859135) + | > loss_disc_real_5: 0.22467924654483795 (0.2391059200710325) + | > loss_0: 2.9786577224731445 (2.7335864252118887) + | > grad_norm_0: tensor(9.9913, device='cuda:0') (tensor(10.7148, device='cuda:0')) + | > loss_gen: 1.8172944784164429 (1.8926532262593359) + | > loss_kl: 2.3846001625061035 (2.5254652784831513) + | > loss_feat: 3.3131730556488037 (3.483379473733665) + | > loss_mel: 21.74222183227539 (21.159920369807743) + | > loss_duration: 1.4162235260009766 (1.3631226728804668) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.673513412475586 (30.424541055859617) + | > grad_norm_1: tensor(144.1346, device='cuda:0') (tensor(138.0184, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.9709 (0.8770211644433623) + | > loader_time: 0.0185 (0.013429486336399666) + + + --> TIME: 2024-06-25 07:29:05 -- STEP: 427/591 -- GLOBAL_STEP: 16975 + | > loss_disc: 2.8430864810943604 (2.7396738585041023) + | > loss_disc_real_0: 0.35443228483200073 (0.2140583009295497) + | > loss_disc_real_1: 0.2565143406391144 (0.23111266438659517) + | > loss_disc_real_2: 0.2487647533416748 (0.23702320362282023) + | > loss_disc_real_3: 0.22107607126235962 (0.23539973659314373) + | > loss_disc_real_4: 0.2636440694332123 (0.23447792731068454) + | > loss_disc_real_5: 0.2810913920402527 (0.2386919953086058) + | > loss_0: 2.8430864810943604 (2.7396738585041023) + | > grad_norm_0: tensor(24.6229, device='cuda:0') (tensor(10.6939, device='cuda:0')) + | > loss_gen: 1.826568365097046 (1.8959370968771763) + | > loss_kl: 2.5370683670043945 (2.522372930893018) + | > loss_feat: 3.1354188919067383 (3.4776320451874922) + | > loss_mel: 21.344932556152344 (21.174022790698853) + | > loss_duration: 1.3811278343200684 (1.3639719519738012) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.225116729736328 (30.433936853877835) + | > grad_norm_1: tensor(183.5167, device='cuda:0') (tensor(144.1536, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.9803 (0.883265730368728) + | > loader_time: 0.0161 (0.013691411364553126) + + + --> TIME: 2024-06-25 07:29:31 -- STEP: 452/591 -- GLOBAL_STEP: 17000 + | > loss_disc: 2.710442543029785 (2.7413380473060913) + | > loss_disc_real_0: 0.17903849482536316 (0.21374160110686732) + | > loss_disc_real_1: 0.26155781745910645 (0.2315058147419343) + | > loss_disc_real_2: 0.23699314892292023 (0.23704240349146116) + | > loss_disc_real_3: 0.2859303057193756 (0.2354120212912032) + | > loss_disc_real_4: 0.22541017830371857 (0.2346969154226569) + | > loss_disc_real_5: 0.2526196539402008 (0.23918035929709408) + | > loss_0: 2.710442543029785 (2.7413380473060913) + | > grad_norm_0: tensor(8.0789, device='cuda:0') (tensor(10.6459, device='cuda:0')) + | > loss_gen: 1.975258708000183 (1.8931463558589463) + | > loss_kl: 2.4812562465667725 (2.518015897906989) + | > loss_feat: 3.4490396976470947 (3.460129708315419) + | > loss_mel: 20.458759307861328 (21.15946430020626) + | > loss_duration: 1.3663735389709473 (1.3650355705645232) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.73068618774414 (30.395791859753363) + | > grad_norm_1: tensor(95.2858, device='cuda:0') (tensor(143.4347, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 0.9973 (0.8897457117527986) + | > loader_time: 0.0229 (0.013927055143677028) + + + --> TIME: 2024-06-25 07:29:57 -- STEP: 477/591 -- GLOBAL_STEP: 17025 + | > loss_disc: 2.7416388988494873 (2.740968619252648) + | > loss_disc_real_0: 0.272219181060791 (0.21382211406463347) + | > loss_disc_real_1: 0.22493180632591248 (0.2311459442354598) + | > loss_disc_real_2: 0.25198233127593994 (0.23694864583965117) + | > loss_disc_real_3: 0.2372605949640274 (0.23539429812436333) + | > loss_disc_real_4: 0.23840251564979553 (0.234811839536301) + | > loss_disc_real_5: 0.2487737387418747 (0.2393672564994364) + | > loss_0: 2.7416388988494873 (2.740968619252648) + | > grad_norm_0: tensor(9.8432, device='cuda:0') (tensor(10.6310, device='cuda:0')) + | > loss_gen: 1.7172236442565918 (1.8914589442047183) + | > loss_kl: 2.4049766063690186 (2.515826023849553) + | > loss_feat: 3.2913665771484375 (3.4491924499815614) + | > loss_mel: 21.02613639831543 (21.140313398413173) + | > loss_duration: 1.385998249053955 (1.3662557027125009) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.825702667236328 (30.363046550150937) + | > grad_norm_1: tensor(140.5200, device='cuda:0') (tensor(140.3064, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 1.0408 (0.8964750286918015) + | > loader_time: 0.016 (0.014170560696839786) + + + --> TIME: 2024-06-25 07:30:24 -- STEP: 502/591 -- GLOBAL_STEP: 17050 + | > loss_disc: 2.8376126289367676 (2.740116919179359) + | > loss_disc_real_0: 0.23111367225646973 (0.21357140878579056) + | > loss_disc_real_1: 0.21462026238441467 (0.23090727377935236) + | > loss_disc_real_2: 0.2409822940826416 (0.23677450201782096) + | > loss_disc_real_3: 0.16803745925426483 (0.2352553196993482) + | > loss_disc_real_4: 0.2150857150554657 (0.23469247606646018) + | > loss_disc_real_5: 0.2133300006389618 (0.23930471708100157) + | > loss_0: 2.8376126289367676 (2.740116919179359) + | > grad_norm_0: tensor(8.6297, device='cuda:0') (tensor(10.5218, device='cuda:0')) + | > loss_gen: 1.9199638366699219 (1.8910891534797698) + | > loss_kl: 2.587841272354126 (2.515095914977482) + | > loss_feat: 3.1541929244995117 (3.442727204338013) + | > loss_mel: 20.721338272094727 (21.12657058001513) + | > loss_duration: 1.4035605192184448 (1.3675647810160876) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.786895751953125 (30.34304767015921) + | > grad_norm_1: tensor(81.9605, device='cuda:0') (tensor(140.1985, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 1.0858 (0.9045133619194485) + | > loader_time: 0.0171 (0.01443574343069616) + + + --> TIME: 2024-06-25 07:30:52 -- STEP: 527/591 -- GLOBAL_STEP: 17075 + | > loss_disc: 2.7473864555358887 (2.7405359500725734) + | > loss_disc_real_0: 0.22649329900741577 (0.21375624510323527) + | > loss_disc_real_1: 0.24664384126663208 (0.23076179456778675) + | > loss_disc_real_2: 0.1903267353773117 (0.23671623863689814) + | > loss_disc_real_3: 0.1778588891029358 (0.23551247898043898) + | > loss_disc_real_4: 0.23710285127162933 (0.2347400621908666) + | > loss_disc_real_5: 0.2509145736694336 (0.23928841891958322) + | > loss_0: 2.7473864555358887 (2.7405359500725734) + | > grad_norm_0: tensor(5.3522, device='cuda:0') (tensor(10.6976, device='cuda:0')) + | > loss_gen: 1.8933253288269043 (1.8898480501754007) + | > loss_kl: 2.437988519668579 (2.5135218613061796) + | > loss_feat: 3.341104507446289 (3.434853324401537) + | > loss_mel: 21.70179557800293 (21.11485925345084) + | > loss_duration: 1.3599882125854492 (1.369271566112308) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.734203338623047 (30.32235410136561) + | > grad_norm_1: tensor(72.1939, device='cuda:0') (tensor(138.6488, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 1.1269 (0.9133791801373012) + | > loader_time: 0.0206 (0.014710047665764305) + + + --> TIME: 2024-06-25 07:31:21 -- STEP: 552/591 -- GLOBAL_STEP: 17100 + | > loss_disc: 2.766231060028076 (2.740681301424466) + | > loss_disc_real_0: 0.2756880521774292 (0.21379749746858207) + | > loss_disc_real_1: 0.2657095491886139 (0.23069491480355675) + | > loss_disc_real_2: 0.26425817608833313 (0.23677011274233245) + | > loss_disc_real_3: 0.2831932306289673 (0.23561988347142504) + | > loss_disc_real_4: 0.2223575860261917 (0.23467687101683754) + | > loss_disc_real_5: 0.24818305671215057 (0.2393877097823914) + | > loss_0: 2.766231060028076 (2.740681301424466) + | > grad_norm_0: tensor(12.7953, device='cuda:0') (tensor(10.5771, device='cuda:0')) + | > loss_gen: 1.7929750680923462 (1.888589400096216) + | > loss_kl: 2.4389331340789795 (2.510919373536457) + | > loss_feat: 3.1667439937591553 (3.4265692864639172) + | > loss_mel: 19.867279052734375 (21.103475739990444) + | > loss_duration: 1.4436917304992676 (1.3708438735077342) + | > amp_scaler: 1024.0 (524.0579710144926) + | > loss_1: 28.70962142944336 (30.30039772434511) + | > grad_norm_1: tensor(85.3997, device='cuda:0') (tensor(136.4626, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 1.1127 (0.9233543000359465) + | > loader_time: 0.0272 (0.015025344447813175) + + + --> TIME: 2024-06-25 07:31:51 -- STEP: 577/591 -- GLOBAL_STEP: 17125 + | > loss_disc: 2.737642526626587 (2.7404799110141513) + | > loss_disc_real_0: 0.13197675347328186 (0.2139174901788322) + | > loss_disc_real_1: 0.19849412143230438 (0.23044208045952036) + | > loss_disc_real_2: 0.21605487167835236 (0.2366453389051262) + | > loss_disc_real_3: 0.24241411685943604 (0.2355543098426981) + | > loss_disc_real_4: 0.23952187597751617 (0.23458887238952894) + | > loss_disc_real_5: 0.24521207809448242 (0.2392953059134079) + | > loss_0: 2.737642526626587 (2.7404799110141513) + | > grad_norm_0: tensor(15.6771, device='cuda:0') (tensor(10.6538, device='cuda:0')) + | > loss_gen: 1.9119471311569214 (1.8880162726655163) + | > loss_kl: 2.455950975418091 (2.508894590612509) + | > loss_feat: 3.026353359222412 (3.419432363229152) + | > loss_mel: 20.584976196289062 (21.089949883869227) + | > loss_duration: 1.4202816486358643 (1.372680806411201) + | > amp_scaler: 512.0 (528.8596187175041) + | > loss_1: 29.39950942993164 (30.278973954595767) + | > grad_norm_1: tensor(286.9102, device='cuda:0') (tensor(136.3692, device='cuda:0')) + | > current_lr_0: 0.0001993011799713115 + | > current_lr_1: 0.0001993011799713115 + | > step_time: 1.2104 (0.9347967824654935) + | > loader_time: 0.023 (0.015357869956480984) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035053161864585067 (-0.00017373612586488087) + | > avg_loss_disc: 2.730893003179671 (-0.027004947053625994) + | > avg_loss_disc_real_0: 0.19087965501115678 (+0.013790035659962507) + | > avg_loss_disc_real_1: 0.1812381699998328 (-0.08946597290799971) + | > avg_loss_disc_real_2: 0.2292808674751444 (-0.062158216187294474) + | > avg_loss_disc_real_3: 0.17531140465685663 (-0.11491504818835147) + | > avg_loss_disc_real_4: 0.25212023264550143 (-0.0007852610755473832) + | > avg_loss_disc_real_5: 0.2464036744959811 (-0.010603011605587409) + | > avg_loss_0: 2.730893003179671 (-0.027004947053625994) + | > avg_loss_gen: 1.7087753630698996 (-0.2972600079597303) + | > avg_loss_kl: 2.5309584090050232 (-0.043573293280094294) + | > avg_loss_feat: 3.3191650065969913 (-0.015014404946185334) + | > avg_loss_mel: 21.138197391591167 (+0.04854936802641063) + | > avg_loss_duration: 1.3847859692066273 (-0.007932769491317382) + | > avg_loss_1: 30.08188207098778 (-0.3152309985871007) + + + > EPOCH: 30/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 07:32:17)  + + --> TIME: 2024-06-25 07:32:27 -- STEP: 11/591 -- GLOBAL_STEP: 17150 + | > loss_disc: 2.675612211227417 (2.693244218826294) + | > loss_disc_real_0: 0.1408839225769043 (0.2025982900099321) + | > loss_disc_real_1: 0.2098173201084137 (0.2307977960868315) + | > loss_disc_real_2: 0.2313956618309021 (0.23145161704583603) + | > loss_disc_real_3: 0.201713427901268 (0.24464669688181442) + | > loss_disc_real_4: 0.21035747230052948 (0.23284964263439178) + | > loss_disc_real_5: 0.24149668216705322 (0.24026558345014398) + | > loss_0: 2.675612211227417 (2.693244218826294) + | > grad_norm_0: tensor(14.7762, device='cuda:0') (tensor(7.2002, device='cuda:0')) + | > loss_gen: 2.0294981002807617 (1.9272371205416592) + | > loss_kl: 2.744318723678589 (2.6473351391878994) + | > loss_feat: 3.8664379119873047 (3.7274642424149946) + | > loss_mel: 21.744213104248047 (21.246979800137606) + | > loss_duration: 1.2982409000396729 (1.339191664348949) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.682708740234375 (30.888208389282227) + | > grad_norm_1: tensor(128.3508, device='cuda:0') (tensor(105.2791, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.793 (0.7700730670582164) + | > loader_time: 0.01 (0.008920561183582653) + + + --> TIME: 2024-06-25 07:32:47 -- STEP: 36/591 -- GLOBAL_STEP: 17175 + | > loss_disc: 2.7265796661376953 (2.716555615266164) + | > loss_disc_real_0: 0.19680199027061462 (0.2106056565211879) + | > loss_disc_real_1: 0.2646152675151825 (0.22874641832378176) + | > loss_disc_real_2: 0.26575538516044617 (0.2335861445301109) + | > loss_disc_real_3: 0.22345955669879913 (0.23619510150618023) + | > loss_disc_real_4: 0.2255057990550995 (0.23571252988444436) + | > loss_disc_real_5: 0.2412625551223755 (0.24017669094933403) + | > loss_0: 2.7265796661376953 (2.716555615266164) + | > grad_norm_0: tensor(3.9026, device='cuda:0') (tensor(9.0384, device='cuda:0')) + | > loss_gen: 1.8102940320968628 (1.911361661222246) + | > loss_kl: 2.6605727672576904 (2.6230852802594504) + | > loss_feat: 3.3252084255218506 (3.619703895515866) + | > loss_mel: 21.343416213989258 (21.258837276034885) + | > loss_duration: 1.3799467086791992 (1.3405681947867076) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.51943588256836 (30.753556304507786) + | > grad_norm_1: tensor(81.7060, device='cuda:0') (tensor(116.4567, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.807 (0.7856006423632304) + | > loader_time: 0.0096 (0.0095848904715644) + + + --> TIME: 2024-06-25 07:33:08 -- STEP: 61/591 -- GLOBAL_STEP: 17200 + | > loss_disc: 2.725376844406128 (2.72052912243077) + | > loss_disc_real_0: 0.12875115871429443 (0.2101716076741453) + | > loss_disc_real_1: 0.22652886807918549 (0.22888302607614486) + | > loss_disc_real_2: 0.23492060601711273 (0.23333025125206494) + | > loss_disc_real_3: 0.1978461593389511 (0.23556486358408069) + | > loss_disc_real_4: 0.2374441921710968 (0.23534984349227342) + | > loss_disc_real_5: 0.2262372374534607 (0.23999104978608304) + | > loss_0: 2.725376844406128 (2.72052912243077) + | > grad_norm_0: tensor(19.5259, device='cuda:0') (tensor(9.4151, device='cuda:0')) + | > loss_gen: 1.9155339002609253 (1.9071412633676998) + | > loss_kl: 2.6181745529174805 (2.6013259184165074) + | > loss_feat: 3.4954159259796143 (3.5829607306933795) + | > loss_mel: 20.716548919677734 (21.174938201904297) + | > loss_duration: 1.3574538230895996 (1.3484775472859865) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.103126525878906 (30.614843649942365) + | > grad_norm_1: tensor(167.6704, device='cuda:0') (tensor(144.7973, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.8287 (0.7959424394075987) + | > loader_time: 0.0102 (0.00985530165375256) + + + --> TIME: 2024-06-25 07:33:29 -- STEP: 86/591 -- GLOBAL_STEP: 17225 + | > loss_disc: 2.72568678855896 (2.7202533982520887) + | > loss_disc_real_0: 0.27061936259269714 (0.21127344529296077) + | > loss_disc_real_1: 0.24085883796215057 (0.22763359598642172) + | > loss_disc_real_2: 0.2421746850013733 (0.2335710574028104) + | > loss_disc_real_3: 0.2521631717681885 (0.23553460039371668) + | > loss_disc_real_4: 0.21628199517726898 (0.23472335899985114) + | > loss_disc_real_5: 0.2364545613527298 (0.23890939008357914) + | > loss_0: 2.72568678855896 (2.7202533982520887) + | > grad_norm_0: tensor(13.6090, device='cuda:0') (tensor(9.7337, device='cuda:0')) + | > loss_gen: 1.855597734451294 (1.9022597615108934) + | > loss_kl: 2.771169900894165 (2.5879455111747567) + | > loss_feat: 3.3932838439941406 (3.5673440057177874) + | > loss_mel: 21.2919979095459 (21.158498475717938) + | > loss_duration: 1.3733572959899902 (1.3549671575080513) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.685407638549805 (30.57101489222327) + | > grad_norm_1: tensor(67.8895, device='cuda:0') (tensor(136.6284, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.8295 (0.8035970316376794) + | > loader_time: 0.0102 (0.010067565496577774) + + + --> TIME: 2024-06-25 07:33:50 -- STEP: 111/591 -- GLOBAL_STEP: 17250 + | > loss_disc: 2.751180648803711 (2.7221099879290613) + | > loss_disc_real_0: 0.21855276823043823 (0.2115758878959192) + | > loss_disc_real_1: 0.20278920233249664 (0.2271618496727299) + | > loss_disc_real_2: 0.21283528208732605 (0.23352285145639298) + | > loss_disc_real_3: 0.20698483288288116 (0.23536703191898964) + | > loss_disc_real_4: 0.22991879284381866 (0.23460945379626644) + | > loss_disc_real_5: 0.2246980220079422 (0.23897854178338437) + | > loss_0: 2.751180648803711 (2.7221099879290613) + | > grad_norm_0: tensor(3.9813, device='cuda:0') (tensor(9.7911, device='cuda:0')) + | > loss_gen: 1.8560795783996582 (1.8987910715309348) + | > loss_kl: 2.547151803970337 (2.5730020076304942) + | > loss_feat: 3.6493337154388428 (3.5656478791623503) + | > loss_mel: 21.701974868774414 (21.15430127393018) + | > loss_duration: 1.3426238298416138 (1.3592435508160972) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.097164154052734 (30.550985697153454) + | > grad_norm_1: tensor(143.8733, device='cuda:0') (tensor(139.6190, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.8573 (0.8103388150533037) + | > loader_time: 0.0116 (0.010329880155958572) + + + --> TIME: 2024-06-25 07:34:11 -- STEP: 136/591 -- GLOBAL_STEP: 17275 + | > loss_disc: 2.689542770385742 (2.72333098860348) + | > loss_disc_real_0: 0.16772134602069855 (0.21142197887906256) + | > loss_disc_real_1: 0.21698148548603058 (0.2274105679243803) + | > loss_disc_real_2: 0.1941288709640503 (0.23426300287246704) + | > loss_disc_real_3: 0.25587207078933716 (0.2354726891088135) + | > loss_disc_real_4: 0.22991527616977692 (0.23419897302108653) + | > loss_disc_real_5: 0.2284262329339981 (0.23929457092548118) + | > loss_0: 2.689542770385742 (2.72333098860348) + | > grad_norm_0: tensor(9.1798, device='cuda:0') (tensor(9.7931, device='cuda:0')) + | > loss_gen: 2.001591920852661 (1.900049049188109) + | > loss_kl: 2.6547162532806396 (2.5642583703293518) + | > loss_feat: 3.857306480407715 (3.5697990340345047) + | > loss_mel: 21.682952880859375 (21.193666233735925) + | > loss_duration: 1.3529062271118164 (1.3600499822812917) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.549476623535156 (30.587822675704956) + | > grad_norm_1: tensor(201.4557, device='cuda:0') (tensor(131.1070, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.8485 (0.8164742308504439) + | > loader_time: 0.0117 (0.01057154290816363) + + + --> TIME: 2024-06-25 07:34:33 -- STEP: 161/591 -- GLOBAL_STEP: 17300 + | > loss_disc: 2.701232433319092 (2.724289763788259) + | > loss_disc_real_0: 0.16351251304149628 (0.21071806226087653) + | > loss_disc_real_1: 0.24548393487930298 (0.2278905704339839) + | > loss_disc_real_2: 0.23464249074459076 (0.23522879775636685) + | > loss_disc_real_3: 0.24344229698181152 (0.23542010330635568) + | > loss_disc_real_4: 0.22748655080795288 (0.23403065906178136) + | > loss_disc_real_5: 0.24402157962322235 (0.2390458732843399) + | > loss_0: 2.701232433319092 (2.724289763788259) + | > grad_norm_0: tensor(13.5371, device='cuda:0') (tensor(9.7798, device='cuda:0')) + | > loss_gen: 1.9698688983917236 (1.8990655271162897) + | > loss_kl: 2.493462324142456 (2.5498023432974497) + | > loss_feat: 3.3549318313598633 (3.5637216271821015) + | > loss_mel: 21.13583755493164 (21.196726200743484) + | > loss_duration: 1.3747568130493164 (1.3581494485369383) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.328857421875 (30.567465089122702) + | > grad_norm_1: tensor(203.5006, device='cuda:0') (tensor(129.4419, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.8579 (0.8227025588847093) + | > loader_time: 0.0113 (0.010871765776450586) + + + --> TIME: 2024-06-25 07:34:56 -- STEP: 186/591 -- GLOBAL_STEP: 17325 + | > loss_disc: 2.6772348880767822 (2.7238628672015293) + | > loss_disc_real_0: 0.22058454155921936 (0.21079076834583796) + | > loss_disc_real_1: 0.2517472505569458 (0.22804265741699487) + | > loss_disc_real_2: 0.23052653670310974 (0.23514247341181643) + | > loss_disc_real_3: 0.21571223437786102 (0.2354091648613253) + | > loss_disc_real_4: 0.2282644808292389 (0.23393815090899828) + | > loss_disc_real_5: 0.24400076270103455 (0.23884499681893215) + | > loss_0: 2.6772348880767822 (2.7238628672015293) + | > grad_norm_0: tensor(5.4639, device='cuda:0') (tensor(9.8546, device='cuda:0')) + | > loss_gen: 1.9193159341812134 (1.900161830327844) + | > loss_kl: 2.381141424179077 (2.5451281186073054) + | > loss_feat: 3.491957664489746 (3.564309254769356) + | > loss_mel: 20.728782653808594 (21.204914134035814) + | > loss_duration: 1.3534953594207764 (1.356883356007197) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.874692916870117 (30.571396643115627) + | > grad_norm_1: tensor(176.8468, device='cuda:0') (tensor(135.5189, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.86 (0.8301516630316292) + | > loader_time: 0.0117 (0.01110740374493342) + + + --> TIME: 2024-06-25 07:35:18 -- STEP: 211/591 -- GLOBAL_STEP: 17350 + | > loss_disc: 2.730168342590332 (2.7242584804788033) + | > loss_disc_real_0: 0.14516088366508484 (0.2113019275651159) + | > loss_disc_real_1: 0.17673753201961517 (0.22740232499572338) + | > loss_disc_real_2: 0.21716095507144928 (0.23533287079413356) + | > loss_disc_real_3: 0.25917667150497437 (0.23562175494517196) + | > loss_disc_real_4: 0.2367485612630844 (0.2337535230610608) + | > loss_disc_real_5: 0.24259218573570251 (0.23877571516127383) + | > loss_0: 2.730168342590332 (2.7242584804788033) + | > grad_norm_0: tensor(14.3821, device='cuda:0') (tensor(10.3925, device='cuda:0')) + | > loss_gen: 1.9599703550338745 (1.9009344509992554) + | > loss_kl: 2.2423810958862305 (2.5355353445803375) + | > loss_feat: 3.5998196601867676 (3.5653805190353034) + | > loss_mel: 21.221115112304688 (21.19370701866691) + | > loss_duration: 1.340014934539795 (1.3564978092202649) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.36330223083496 (30.552055114818412) + | > grad_norm_1: tensor(55.7686, device='cuda:0') (tensor(132.9770, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.8819 (0.8356461197279075) + | > loader_time: 0.0145 (0.011388648742748093) + + + --> TIME: 2024-06-25 07:35:40 -- STEP: 236/591 -- GLOBAL_STEP: 17375 + | > loss_disc: 2.6034133434295654 (2.7233873413781) + | > loss_disc_real_0: 0.17908060550689697 (0.21125569601811595) + | > loss_disc_real_1: 0.24121873080730438 (0.227457713398893) + | > loss_disc_real_2: 0.20367252826690674 (0.2349572069311546) + | > loss_disc_real_3: 0.2063731700181961 (0.23528062703750902) + | > loss_disc_real_4: 0.20726384222507477 (0.23389324653199164) + | > loss_disc_real_5: 0.22916118800640106 (0.23879724728353954) + | > loss_0: 2.6034133434295654 (2.7233873413781) + | > grad_norm_0: tensor(4.1289, device='cuda:0') (tensor(10.2554, device='cuda:0')) + | > loss_gen: 2.0476608276367188 (1.9007359035944535) + | > loss_kl: 2.38694429397583 (2.532254626185206) + | > loss_feat: 3.6493728160858154 (3.5630085882494007) + | > loss_mel: 21.891525268554688 (21.192781416036304) + | > loss_duration: 1.3680850267410278 (1.3566934441105798) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.34358787536621 (30.5454739392814) + | > grad_norm_1: tensor(200.5832, device='cuda:0') (tensor(130.9374, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.9128 (0.8405556608054596) + | > loader_time: 0.0148 (0.011703486159696413) + + + --> TIME: 2024-06-25 07:36:03 -- STEP: 261/591 -- GLOBAL_STEP: 17400 + | > loss_disc: 2.6685357093811035 (2.7215725763547476) + | > loss_disc_real_0: 0.227828711271286 (0.21116481276079155) + | > loss_disc_real_1: 0.21262337267398834 (0.22732530779765484) + | > loss_disc_real_2: 0.20767590403556824 (0.23480454150059213) + | > loss_disc_real_3: 0.24971145391464233 (0.2354673211716144) + | > loss_disc_real_4: 0.23146434128284454 (0.23366105385210323) + | > loss_disc_real_5: 0.22949041426181793 (0.23877720711788455) + | > loss_0: 2.6685357093811035 (2.7215725763547476) + | > grad_norm_0: tensor(4.5063, device='cuda:0') (tensor(10.1020, device='cuda:0')) + | > loss_gen: 1.792826533317566 (1.9013539931783274) + | > loss_kl: 2.659032106399536 (2.532015321811953) + | > loss_feat: 3.5000686645507812 (3.5639442657602243) + | > loss_mel: 20.545425415039062 (21.186913859341782) + | > loss_duration: 1.3820600509643555 (1.3570609142953869) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.879409790039062 (30.541288324699547) + | > grad_norm_1: tensor(228.5202, device='cuda:0') (tensor(133.6950, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.8958 (0.8461567515157647) + | > loader_time: 0.0158 (0.011964717587291963) + + + --> TIME: 2024-06-25 07:36:27 -- STEP: 286/591 -- GLOBAL_STEP: 17425 + | > loss_disc: 2.68428373336792 (2.7218144640222306) + | > loss_disc_real_0: 0.17970117926597595 (0.21088759027994597) + | > loss_disc_real_1: 0.1759035438299179 (0.2272212432397829) + | > loss_disc_real_2: 0.23812130093574524 (0.2349103206208536) + | > loss_disc_real_3: 0.21295706927776337 (0.23550127853046765) + | > loss_disc_real_4: 0.23219187557697296 (0.23364317635973017) + | > loss_disc_real_5: 0.23332050442695618 (0.23885233731536598) + | > loss_0: 2.68428373336792 (2.7218144640222306) + | > grad_norm_0: tensor(8.6482, device='cuda:0') (tensor(10.0120, device='cuda:0')) + | > loss_gen: 1.9769911766052246 (1.9010767603253984) + | > loss_kl: 2.513308525085449 (2.5293956586530983) + | > loss_feat: 3.606955051422119 (3.559445236946319) + | > loss_mel: 21.356645584106445 (21.173245750107128) + | > loss_duration: 1.359981656074524 (1.3574999087340351) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.81388282775879 (30.52066329475883) + | > grad_norm_1: tensor(246.1603, device='cuda:0') (tensor(134.1429, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.9079 (0.851701813977915) + | > loader_time: 0.0163 (0.012211821295998312) + + + --> TIME: 2024-06-25 07:36:50 -- STEP: 311/591 -- GLOBAL_STEP: 17450 + | > loss_disc: 2.7092485427856445 (2.7216380593094414) + | > loss_disc_real_0: 0.20416933298110962 (0.2106850105562394) + | > loss_disc_real_1: 0.2532983720302582 (0.2277389171135004) + | > loss_disc_real_2: 0.22891202569007874 (0.23482413920558917) + | > loss_disc_real_3: 0.2300560474395752 (0.2357372447033787) + | > loss_disc_real_4: 0.29255038499832153 (0.23379353512905035) + | > loss_disc_real_5: 0.30466175079345703 (0.23886183139020606) + | > loss_0: 2.7092485427856445 (2.7216380593094414) + | > grad_norm_0: tensor(4.1544, device='cuda:0') (tensor(9.8291, device='cuda:0')) + | > loss_gen: 1.8695851564407349 (1.9023849668993444) + | > loss_kl: 2.5177459716796875 (2.5276212929912694) + | > loss_feat: 3.48420786857605 (3.558719523083359) + | > loss_mel: 21.401094436645508 (21.172066041510966) + | > loss_duration: 1.3364840745925903 (1.358075395274393) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.609119415283203 (30.518867198294) + | > grad_norm_1: tensor(67.9955, device='cuda:0') (tensor(134.8066, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.9308 (0.8575344614660625) + | > loader_time: 0.0162 (0.012457396056491073) + + + --> TIME: 2024-06-25 07:37:14 -- STEP: 336/591 -- GLOBAL_STEP: 17475 + | > loss_disc: 2.760195732116699 (2.723380833864211) + | > loss_disc_real_0: 0.2699011564254761 (0.21107152315034045) + | > loss_disc_real_1: 0.24926432967185974 (0.22765093247982718) + | > loss_disc_real_2: 0.2478632926940918 (0.23533980335508073) + | > loss_disc_real_3: 0.2364995777606964 (0.23576692842124475) + | > loss_disc_real_4: 0.22064335644245148 (0.2335654730420737) + | > loss_disc_real_5: 0.2303113341331482 (0.23876617476344109) + | > loss_0: 2.760195732116699 (2.723380833864211) + | > grad_norm_0: tensor(12.8152, device='cuda:0') (tensor(9.9950, device='cuda:0')) + | > loss_gen: 1.8131022453308105 (1.9019736108325778) + | > loss_kl: 2.4646832942962646 (2.524027262415205) + | > loss_feat: 3.2769970893859863 (3.5551690657933555) + | > loss_mel: 20.788013458251953 (21.18097284294309) + | > loss_duration: 1.3652113676071167 (1.3590833484416922) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.7080078125 (30.521226116589137) + | > grad_norm_1: tensor(144.2550, device='cuda:0') (tensor(133.5089, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.9563 (0.8632690360148748) + | > loader_time: 0.017 (0.012718159528005686) + + + --> TIME: 2024-06-25 07:37:38 -- STEP: 361/591 -- GLOBAL_STEP: 17500 + | > loss_disc: 2.744683265686035 (2.72428965106235) + | > loss_disc_real_0: 0.3172008991241455 (0.21113991821984504) + | > loss_disc_real_1: 0.24623383581638336 (0.22770293056964874) + | > loss_disc_real_2: 0.2663401961326599 (0.23564696724725231) + | > loss_disc_real_3: 0.2649598717689514 (0.2358902920233576) + | > loss_disc_real_4: 0.23506343364715576 (0.2336242480043559) + | > loss_disc_real_5: 0.23700092732906342 (0.2387819149405012) + | > loss_0: 2.744683265686035 (2.72428965106235) + | > grad_norm_0: tensor(15.8219, device='cuda:0') (tensor(10.0191, device='cuda:0')) + | > loss_gen: 1.7454092502593994 (1.9017714886123784) + | > loss_kl: 2.4614298343658447 (2.521939640890528) + | > loss_feat: 3.419938087463379 (3.550274755155611) + | > loss_mel: 21.014102935791016 (21.161468696065885) + | > loss_duration: 1.448743462562561 (1.3607015794664212) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.089622497558594 (30.496156143019405) + | > grad_norm_1: tensor(335.2014, device='cuda:0') (tensor(134.4019, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.9539 (0.8686915712013139) + | > loader_time: 0.0187 (0.012963156951101198) + + + --> TIME: 2024-06-25 07:38:03 -- STEP: 386/591 -- GLOBAL_STEP: 17525 + | > loss_disc: 2.7116241455078125 (2.7239804471712667) + | > loss_disc_real_0: 0.3089376986026764 (0.21098227800868954) + | > loss_disc_real_1: 0.19623063504695892 (0.22776372282925048) + | > loss_disc_real_2: 0.25746092200279236 (0.2357085538651659) + | > loss_disc_real_3: 0.18998600542545319 (0.2355306981604334) + | > loss_disc_real_4: 0.2249935418367386 (0.2335882246339877) + | > loss_disc_real_5: 0.24099214375019073 (0.23877040296792984) + | > loss_0: 2.7116241455078125 (2.7239804471712667) + | > grad_norm_0: tensor(15.4846, device='cuda:0') (tensor(10.1602, device='cuda:0')) + | > loss_gen: 1.846247673034668 (1.9014182692982373) + | > loss_kl: 2.3481554985046387 (2.520731951906274) + | > loss_feat: 3.5045881271362305 (3.5457150306108702) + | > loss_mel: 21.24462890625 (21.14761289290195) + | > loss_duration: 1.3676435947418213 (1.3620766580413666) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.311262130737305 (30.477554770948974) + | > grad_norm_1: tensor(149.4554, device='cuda:0') (tensor(137.0937, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.9709 (0.8746726185546638) + | > loader_time: 0.018 (0.013213388660411138) + + + --> TIME: 2024-06-25 07:38:28 -- STEP: 411/591 -- GLOBAL_STEP: 17550 + | > loss_disc: 2.718730926513672 (2.723642244246175) + | > loss_disc_real_0: 0.1678457260131836 (0.21080086775432247) + | > loss_disc_real_1: 0.23076723515987396 (0.22753567087244234) + | > loss_disc_real_2: 0.24251040816307068 (0.23570775768182572) + | > loss_disc_real_3: 0.24928376078605652 (0.235677588051253) + | > loss_disc_real_4: 0.23892591893672943 (0.23355250923447946) + | > loss_disc_real_5: 0.23916438221931458 (0.23884127330751023) + | > loss_0: 2.718730926513672 (2.723642244246175) + | > grad_norm_0: tensor(9.9267, device='cuda:0') (tensor(10.0481, device='cuda:0')) + | > loss_gen: 1.8347935676574707 (1.9005969330632193) + | > loss_kl: 2.5467519760131836 (2.517510100872847) + | > loss_feat: 3.417940139770508 (3.5397348328518463) + | > loss_mel: 20.737627029418945 (21.141900357248712) + | > loss_duration: 1.3855518102645874 (1.3630226580766) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.922664642333984 (30.462764832805252) + | > grad_norm_1: tensor(125.7238, device='cuda:0') (tensor(135.1939, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 0.9927 (0.8807584102426422) + | > loader_time: 0.0155 (0.01347892823880606) + + + --> TIME: 2024-06-25 07:38:53 -- STEP: 436/591 -- GLOBAL_STEP: 17575 + | > loss_disc: 2.7664706707000732 (2.724628379585547) + | > loss_disc_real_0: 0.2506732940673828 (0.21101605427374534) + | > loss_disc_real_1: 0.19959457218647003 (0.22746924936361268) + | > loss_disc_real_2: 0.2246805876493454 (0.2358297381310835) + | > loss_disc_real_3: 0.251924991607666 (0.23596237582761212) + | > loss_disc_real_4: 0.23313230276107788 (0.23348291266128557) + | > loss_disc_real_5: 0.20799514651298523 (0.23870224273259485) + | > loss_0: 2.7664706707000732 (2.724628379585547) + | > grad_norm_0: tensor(9.0522, device='cuda:0') (tensor(10.3196, device='cuda:0')) + | > loss_gen: 1.7726109027862549 (1.9006111493351263) + | > loss_kl: 2.3874475955963135 (2.516372998373224) + | > loss_feat: 3.419497489929199 (3.53406228931672) + | > loss_mel: 21.53338623046875 (21.135220396409334) + | > loss_duration: 1.3987088203430176 (1.3639093115242256) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.51165199279785 (30.45017609902478) + | > grad_norm_1: tensor(162.0033, device='cuda:0') (tensor(137.1815, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 1.0123 (0.8869200933963881) + | > loader_time: 0.0186 (0.01374746626670207) + + + --> TIME: 2024-06-25 07:39:19 -- STEP: 461/591 -- GLOBAL_STEP: 17600 + | > loss_disc: 2.779196262359619 (2.7250207230739645) + | > loss_disc_real_0: 0.12911981344223022 (0.2109988529938162) + | > loss_disc_real_1: 0.20134909451007843 (0.2276400232974464) + | > loss_disc_real_2: 0.2432982325553894 (0.2359218026901274) + | > loss_disc_real_3: 0.23725557327270508 (0.236000932025806) + | > loss_disc_real_4: 0.22114242613315582 (0.23347768992379533) + | > loss_disc_real_5: 0.2392970323562622 (0.23889694025097596) + | > loss_0: 2.779196262359619 (2.7250207230739645) + | > grad_norm_0: tensor(19.7815, device='cuda:0') (tensor(10.2703, device='cuda:0')) + | > loss_gen: 1.7767940759658813 (1.900202332504918) + | > loss_kl: 2.458338499069214 (2.5153180632311973) + | > loss_feat: 3.2537968158721924 (3.5271196748067397) + | > loss_mel: 20.88807487487793 (21.123577117919915) + | > loss_duration: 1.3886239528656006 (1.364983339888931) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.765628814697266 (30.43120050740604) + | > grad_norm_1: tensor(82.7050, device='cuda:0') (tensor(136.4174, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 1.015 (0.893470567632911) + | > loader_time: 0.0188 (0.014031866367364908) + + + --> TIME: 2024-06-25 07:39:45 -- STEP: 486/591 -- GLOBAL_STEP: 17625 + | > loss_disc: 2.6983890533447266 (2.724703671019755) + | > loss_disc_real_0: 0.27392810583114624 (0.21100461504464288) + | > loss_disc_real_1: 0.20165032148361206 (0.2275391803295524) + | > loss_disc_real_2: 0.23548021912574768 (0.23593176818556255) + | > loss_disc_real_3: 0.26995861530303955 (0.2359179017352469) + | > loss_disc_real_4: 0.23460893332958221 (0.23340623608096636) + | > loss_disc_real_5: 0.2495405226945877 (0.23888308088843224) + | > loss_0: 2.6983890533447266 (2.724703671019755) + | > grad_norm_0: tensor(14.7938, device='cuda:0') (tensor(10.1501, device='cuda:0')) + | > loss_gen: 1.7600983381271362 (1.8985477798760182) + | > loss_kl: 2.4498767852783203 (2.512703611036387) + | > loss_feat: 3.2768945693969727 (3.517318670641738) + | > loss_mel: 19.801267623901367 (21.104629508752385) + | > loss_duration: 1.3729677200317383 (1.366374949123634) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 28.661102294921875 (30.399574495637367) + | > grad_norm_1: tensor(65.4715, device='cuda:0') (tensor(135.8354, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 1.0656 (0.9008582575331009) + | > loader_time: 0.0221 (0.014337681939081887) + + + --> TIME: 2024-06-25 07:40:13 -- STEP: 511/591 -- GLOBAL_STEP: 17650 + | > loss_disc: 2.6877963542938232 (2.7257040970012874) + | > loss_disc_real_0: 0.17489375174045563 (0.21108643725193643) + | > loss_disc_real_1: 0.22309239208698273 (0.22756947754880452) + | > loss_disc_real_2: 0.22621707618236542 (0.23584961929083337) + | > loss_disc_real_3: 0.23767860233783722 (0.236335192203755) + | > loss_disc_real_4: 0.22166383266448975 (0.23339724899388106) + | > loss_disc_real_5: 0.23076705634593964 (0.23898495780395201) + | > loss_0: 2.6877963542938232 (2.7257040970012874) + | > grad_norm_0: tensor(8.6390, device='cuda:0') (tensor(10.2883, device='cuda:0')) + | > loss_gen: 1.9317078590393066 (1.897949503592549) + | > loss_kl: 2.558598756790161 (2.512212545204536) + | > loss_feat: 3.5789599418640137 (3.5092952899970187) + | > loss_mel: 20.86844253540039 (21.095458614849523) + | > loss_duration: 1.413206696510315 (1.3678834016075812) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.350914001464844 (30.38279931652569) + | > grad_norm_1: tensor(140.3751, device='cuda:0') (tensor(134.1629, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 1.0638 (0.9095428633829852) + | > loader_time: 0.0192 (0.014645712249666274) + + + --> TIME: 2024-06-25 07:40:42 -- STEP: 536/591 -- GLOBAL_STEP: 17675 + | > loss_disc: 2.7226638793945312 (2.7260676800315067) + | > loss_disc_real_0: 0.3007141649723053 (0.21128475393599536) + | > loss_disc_real_1: 0.2467859834432602 (0.22762572159295652) + | > loss_disc_real_2: 0.23469099402427673 (0.2358739723020525) + | > loss_disc_real_3: 0.23720155656337738 (0.23624345309921163) + | > loss_disc_real_4: 0.2678576707839966 (0.23342491135890805) + | > loss_disc_real_5: 0.2779073417186737 (0.23906399979631401) + | > loss_0: 2.7226638793945312 (2.7260676800315067) + | > grad_norm_0: tensor(14.8426, device='cuda:0') (tensor(10.3486, device='cuda:0')) + | > loss_gen: 1.8648368120193481 (1.8975514780674407) + | > loss_kl: 2.368666172027588 (2.509966053624651) + | > loss_feat: 3.381861448287964 (3.502380165146358) + | > loss_mel: 20.920089721679688 (21.079555966960843) + | > loss_duration: 1.4159146547317505 (1.3694107314543942) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.95136833190918 (30.358864367897834) + | > grad_norm_1: tensor(82.6471, device='cuda:0') (tensor(132.4088, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 1.1177 (0.9198346293684262) + | > loader_time: 0.02 (0.014989570895237706) + + + --> TIME: 2024-06-25 07:41:12 -- STEP: 561/591 -- GLOBAL_STEP: 17700 + | > loss_disc: 2.798825263977051 (2.726217026379021) + | > loss_disc_real_0: 0.20395562052726746 (0.21130119396934746) + | > loss_disc_real_1: 0.2445029765367508 (0.2275731331418117) + | > loss_disc_real_2: 0.25695744156837463 (0.23574979799847764) + | > loss_disc_real_3: 0.2642262876033783 (0.23623686776547934) + | > loss_disc_real_4: 0.2551160752773285 (0.23339305138099214) + | > loss_disc_real_5: 0.24998098611831665 (0.23904470461893845) + | > loss_0: 2.798825263977051 (2.726217026379021) + | > grad_norm_0: tensor(5.3716, device='cuda:0') (tensor(10.3624, device='cuda:0')) + | > loss_gen: 1.8714301586151123 (1.8965860634989067) + | > loss_kl: 2.4646975994110107 (2.5073021036205865) + | > loss_feat: 2.9667885303497314 (3.4939830073698315) + | > loss_mel: 20.206377029418945 (21.05131556601021) + | > loss_duration: 1.3567168712615967 (1.3709583832830887) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 28.866008758544922 (30.320145098708316) + | > grad_norm_1: tensor(98.7025, device='cuda:0') (tensor(132.1138, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 1.2043 (0.9308453796169125) + | > loader_time: 0.0257 (0.015350306947813523) + + + --> TIME: 2024-06-25 07:41:43 -- STEP: 586/591 -- GLOBAL_STEP: 17725 + | > loss_disc: 2.7035140991210938 (2.726019685178894) + | > loss_disc_real_0: 0.18832409381866455 (0.2112024818950749) + | > loss_disc_real_1: 0.2482934594154358 (0.22759202895717817) + | > loss_disc_real_2: 0.2396165132522583 (0.2356282073127125) + | > loss_disc_real_3: 0.2458544671535492 (0.23612122995454704) + | > loss_disc_real_4: 0.26279211044311523 (0.23342227254304462) + | > loss_disc_real_5: 0.24957218766212463 (0.23909658531788672) + | > loss_0: 2.7035140991210938 (2.726019685178894) + | > grad_norm_0: tensor(5.9424, device='cuda:0') (tensor(10.3702, device='cuda:0')) + | > loss_gen: 1.8928468227386475 (1.8959954031091502) + | > loss_kl: 2.3578686714172363 (2.504822547525269) + | > loss_feat: 3.4102115631103516 (3.48783553663781) + | > loss_mel: 20.681777954101562 (21.03614267798412) + | > loss_duration: 1.3893669843673706 (1.372556284019159) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.732070922851562 (30.297352416523488) + | > grad_norm_1: tensor(78.1744, device='cuda:0') (tensor(132.2534, device='cuda:0')) + | > current_lr_0: 0.00019927626732381507 + | > current_lr_1: 0.00019927626732381507 + | > step_time: 1.2276 (0.9426361074219791) + | > loader_time: 0.0219 (0.015738396921255476) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035808745850907995 (+7.555839863229285e-05) + | > avg_loss_disc: 2.7154712271183095 (-0.015421776061361747) + | > avg_loss_disc_real_0: 0.16809670611264857 (-0.02278294889850821) + | > avg_loss_disc_real_1: 0.20025729277032486 (+0.01901912277049206) + | > avg_loss_disc_real_2: 0.24540652302985497 (+0.016125655554710572) + | > avg_loss_disc_real_3: 0.18518628719005178 (+0.009874882533195156) + | > avg_loss_disc_real_4: 0.23889342013825762 (-0.013226812507243807) + | > avg_loss_disc_real_5: 0.23854468001964244 (-0.007858994476338671) + | > avg_loss_0: 2.7154712271183095 (-0.015421776061361747) + | > avg_loss_gen: 1.7467648957638031 (+0.03798953269390348) + | > avg_loss_kl: 2.57326455826455 (+0.042306149259526915) + | > avg_loss_feat: 3.4531157067481506 (+0.13395070015115929) + | > avg_loss_mel: 20.84641744735393 (-0.29177994423723774) + | > avg_loss_duration: 1.4093781506761598 (+0.02459218146953246) + | > avg_loss_1: 30.028940566042635 (-0.05294150494514582) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_17730.pth + + > EPOCH: 31/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 07:42:01)  + + --> TIME: 2024-06-25 07:42:19 -- STEP: 20/591 -- GLOBAL_STEP: 17750 + | > loss_disc: 2.713514566421509 (2.7040096044540407) + | > loss_disc_real_0: 0.27568137645721436 (0.21044458039104938) + | > loss_disc_real_1: 0.24623435735702515 (0.2253119133412838) + | > loss_disc_real_2: 0.20940706133842468 (0.23175356760621071) + | > loss_disc_real_3: 0.24368737637996674 (0.23852810487151146) + | > loss_disc_real_4: 0.2378930002450943 (0.23601592406630517) + | > loss_disc_real_5: 0.2472120076417923 (0.23955125361680984) + | > loss_0: 2.713514566421509 (2.7040096044540407) + | > grad_norm_0: tensor(10.5724, device='cuda:0') (tensor(8.3054, device='cuda:0')) + | > loss_gen: 1.8853774070739746 (1.9235802173614502) + | > loss_kl: 2.519390344619751 (2.63036128282547) + | > loss_feat: 3.7116403579711914 (3.76023405790329) + | > loss_mel: 21.232608795166016 (21.146894264221192) + | > loss_duration: 1.3309533596038818 (1.340429323911667) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.679969787597656 (30.80149927139282) + | > grad_norm_1: tensor(73.6546, device='cuda:0') (tensor(114.9328, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.7939 (0.77916179895401) + | > loader_time: 0.0089 (0.008826708793640137) + + + --> TIME: 2024-06-25 07:42:39 -- STEP: 45/591 -- GLOBAL_STEP: 17775 + | > loss_disc: 2.730772018432617 (2.7284168402353925) + | > loss_disc_real_0: 0.20602163672447205 (0.21343557784954706) + | > loss_disc_real_1: 0.21064099669456482 (0.2275497363673316) + | > loss_disc_real_2: 0.22727510333061218 (0.23545835746659172) + | > loss_disc_real_3: 0.23172716796398163 (0.23708861635790932) + | > loss_disc_real_4: 0.2470352202653885 (0.2342360642221239) + | > loss_disc_real_5: 0.23574011027812958 (0.23791787723700206) + | > loss_0: 2.730772018432617 (2.7284168402353925) + | > grad_norm_0: tensor(3.4097, device='cuda:0') (tensor(10.5766, device='cuda:0')) + | > loss_gen: 2.0022478103637695 (1.9174230787489148) + | > loss_kl: 2.5477449893951416 (2.6004529529147686) + | > loss_feat: 3.73514723777771 (3.6904676490359836) + | > loss_mel: 21.323760986328125 (21.20061149597168) + | > loss_duration: 1.3642722368240356 (1.3452557007471722) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.973175048828125 (30.754210917154946) + | > grad_norm_1: tensor(135.4510, device='cuda:0') (tensor(121.0322, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.8128 (0.7925871213277181) + | > loader_time: 0.0101 (0.00967504183451335) + + + --> TIME: 2024-06-25 07:43:00 -- STEP: 70/591 -- GLOBAL_STEP: 17800 + | > loss_disc: 2.70460844039917 (2.7253020422799246) + | > loss_disc_real_0: 0.14923244714736938 (0.20998418682387898) + | > loss_disc_real_1: 0.2305804342031479 (0.22914053669997625) + | > loss_disc_real_2: 0.25818532705307007 (0.2365427885736738) + | > loss_disc_real_3: 0.2084846794605255 (0.23580360923494612) + | > loss_disc_real_4: 0.20925652980804443 (0.23510953038930893) + | > loss_disc_real_5: 0.22175233066082 (0.23847181860889707) + | > loss_0: 2.70460844039917 (2.7253020422799246) + | > grad_norm_0: tensor(10.7229, device='cuda:0') (tensor(9.6872, device='cuda:0')) + | > loss_gen: 1.9138264656066895 (1.9129040156091963) + | > loss_kl: 2.644266128540039 (2.5855632816042227) + | > loss_feat: 3.4879798889160156 (3.6594535282679965) + | > loss_mel: 20.956777572631836 (21.204810551234655) + | > loss_duration: 1.3633618354797363 (1.3509511828422547) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.3662109375 (30.71368247440883) + | > grad_norm_1: tensor(185.4523, device='cuda:0') (tensor(113.6480, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.8146 (0.8009410790034703) + | > loader_time: 0.0104 (0.010091539791652138) + + + --> TIME: 2024-06-25 07:43:21 -- STEP: 95/591 -- GLOBAL_STEP: 17825 + | > loss_disc: 2.7774250507354736 (2.729351430190237) + | > loss_disc_real_0: 0.13107451796531677 (0.21023029034074983) + | > loss_disc_real_1: 0.25204235315322876 (0.229177021666577) + | > loss_disc_real_2: 0.20344363152980804 (0.23678979308981646) + | > loss_disc_real_3: 0.27260252833366394 (0.23629582822322845) + | > loss_disc_real_4: 0.25296494364738464 (0.23521215696083872) + | > loss_disc_real_5: 0.2589644491672516 (0.23912544219117415) + | > loss_0: 2.7774250507354736 (2.729351430190237) + | > grad_norm_0: tensor(18.1877, device='cuda:0') (tensor(10.2518, device='cuda:0')) + | > loss_gen: 1.8853182792663574 (1.9101883386310778) + | > loss_kl: 2.7386410236358643 (2.5789820194244393) + | > loss_feat: 3.316091299057007 (3.643918840508712) + | > loss_mel: 21.04679298400879 (21.21504783630372) + | > loss_duration: 1.3726186752319336 (1.358611040366324) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.35946273803711 (30.706747958534642) + | > grad_norm_1: tensor(51.0980, device='cuda:0') (tensor(113.9941, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.8314 (0.8082520836278012) + | > loader_time: 0.0109 (0.010317661887721017) + + + --> TIME: 2024-06-25 07:43:43 -- STEP: 120/591 -- GLOBAL_STEP: 17850 + | > loss_disc: 2.722696304321289 (2.7318348129590353) + | > loss_disc_real_0: 0.28078967332839966 (0.21140143082787594) + | > loss_disc_real_1: 0.27552175521850586 (0.22913417952756088) + | > loss_disc_real_2: 0.22463686764240265 (0.2370211962610483) + | > loss_disc_real_3: 0.24458713829517365 (0.23606789025167624) + | > loss_disc_real_4: 0.227257639169693 (0.2347120234121879) + | > loss_disc_real_5: 0.2239687442779541 (0.23934508288900058) + | > loss_0: 2.722696304321289 (2.7318348129590353) + | > grad_norm_0: tensor(14.9144, device='cuda:0') (tensor(10.2002, device='cuda:0')) + | > loss_gen: 1.8699759244918823 (1.9048575699329375) + | > loss_kl: 2.3950822353363037 (2.5581220964590714) + | > loss_feat: 3.4180185794830322 (3.61648268699646) + | > loss_mel: 21.4468936920166 (21.190706094106044) + | > loss_duration: 1.3780462741851807 (1.3598188181718196) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.50801658630371 (30.629987144470213) + | > grad_norm_1: tensor(44.4321, device='cuda:0') (tensor(111.0222, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.8557 (0.8153668840726217) + | > loader_time: 0.0124 (0.01060722668965658) + + + --> TIME: 2024-06-25 07:44:04 -- STEP: 145/591 -- GLOBAL_STEP: 17875 + | > loss_disc: 2.649080753326416 (2.7264896968315386) + | > loss_disc_real_0: 0.2262817621231079 (0.21087289673501045) + | > loss_disc_real_1: 0.21718621253967285 (0.22841417779182566) + | > loss_disc_real_2: 0.22499871253967285 (0.2364507048294462) + | > loss_disc_real_3: 0.246818408370018 (0.23560492601887933) + | > loss_disc_real_4: 0.2480299472808838 (0.23392144236071358) + | > loss_disc_real_5: 0.22382648289203644 (0.23866083683638736) + | > loss_0: 2.649080753326416 (2.7264896968315386) + | > grad_norm_0: tensor(5.1439, device='cuda:0') (tensor(10.0237, device='cuda:0')) + | > loss_gen: 1.9483599662780762 (1.906114856128035) + | > loss_kl: 2.5898451805114746 (2.5514442937127475) + | > loss_feat: 3.8257570266723633 (3.6208735597544695) + | > loss_mel: 21.280427932739258 (21.167804599630426) + | > loss_duration: 1.3837668895721436 (1.3594063043594364) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.028156280517578 (30.605643528905407) + | > grad_norm_1: tensor(84.0740, device='cuda:0') (tensor(113.0610, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.8442 (0.8208856911494814) + | > loader_time: 0.0115 (0.010794323888318295) + + + --> TIME: 2024-06-25 07:44:26 -- STEP: 170/591 -- GLOBAL_STEP: 17900 + | > loss_disc: 2.8224008083343506 (2.7272770446889547) + | > loss_disc_real_0: 0.4506622552871704 (0.2118969792390571) + | > loss_disc_real_1: 0.23466211557388306 (0.22891129754921968) + | > loss_disc_real_2: 0.26579442620277405 (0.2364787807359415) + | > loss_disc_real_3: 0.24920566380023956 (0.23555834652746424) + | > loss_disc_real_4: 0.19994275271892548 (0.23356186098912182) + | > loss_disc_real_5: 0.23136411607265472 (0.23860607760794023) + | > loss_0: 2.8224008083343506 (2.7272770446889547) + | > grad_norm_0: tensor(35.2587, device='cuda:0') (tensor(10.6458, device='cuda:0')) + | > loss_gen: 1.7982475757598877 (1.9102659940719604) + | > loss_kl: 2.536379337310791 (2.545908385164597) + | > loss_feat: 3.670064926147461 (3.6217422920114846) + | > loss_mel: 21.490659713745117 (21.173576298881986) + | > loss_duration: 1.3418654203414917 (1.3573104655041415) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.837217330932617 (30.608803412493536) + | > grad_norm_1: tensor(120.7201, device='cuda:0') (tensor(127.3291, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.865 (0.827336451586555) + | > loader_time: 0.0115 (0.010967285492840935) + + + --> TIME: 2024-06-25 07:44:49 -- STEP: 195/591 -- GLOBAL_STEP: 17925 + | > loss_disc: 2.708291530609131 (2.7299376157613913) + | > loss_disc_real_0: 0.13871294260025024 (0.2112093931971452) + | > loss_disc_real_1: 0.21670213341712952 (0.22953596390210665) + | > loss_disc_real_2: 0.20646806061267853 (0.23661838410756528) + | > loss_disc_real_3: 0.20751884579658508 (0.2355935812760622) + | > loss_disc_real_4: 0.2005099058151245 (0.23350105644800725) + | > loss_disc_real_5: 0.1861237734556198 (0.2386170642498212) + | > loss_0: 2.708291530609131 (2.7299376157613913) + | > grad_norm_0: tensor(12.4973, device='cuda:0') (tensor(10.4088, device='cuda:0')) + | > loss_gen: 2.0621707439422607 (1.9143763591081668) + | > loss_kl: 2.529888868331909 (2.541352720749684) + | > loss_feat: 3.555243968963623 (3.619087464992816) + | > loss_mel: 21.203306198120117 (21.184538073417478) + | > loss_duration: 1.3469469547271729 (1.3561978340148932) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.697555541992188 (30.615552432720477) + | > grad_norm_1: tensor(128.0020, device='cuda:0') (tensor(139.8353, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.8871 (0.8329312104445238) + | > loader_time: 0.012 (0.011168486032730494) + + + --> TIME: 2024-06-25 07:45:11 -- STEP: 220/591 -- GLOBAL_STEP: 17950 + | > loss_disc: 2.663020133972168 (2.7289237022399915) + | > loss_disc_real_0: 0.2499551624059677 (0.21138699972494082) + | > loss_disc_real_1: 0.23047997057437897 (0.22967306592247702) + | > loss_disc_real_2: 0.25246483087539673 (0.23623096604238858) + | > loss_disc_real_3: 0.2717077136039734 (0.23581913167780097) + | > loss_disc_real_4: 0.22320310771465302 (0.23369190062988887) + | > loss_disc_real_5: 0.24021504819393158 (0.2388558272611011) + | > loss_0: 2.663020133972168 (2.7289237022399915) + | > grad_norm_0: tensor(9.8414, device='cuda:0') (tensor(10.2774, device='cuda:0')) + | > loss_gen: 1.7181962728500366 (1.9113549297506158) + | > loss_kl: 2.454390048980713 (2.5397938034751197) + | > loss_feat: 3.517881393432617 (3.607996093143116) + | > loss_mel: 20.511770248413086 (21.16559933749113) + | > loss_duration: 1.3336870670318604 (1.3552578265016735) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.535924911499023 (30.580001987110485) + | > grad_norm_1: tensor(47.6765, device='cuda:0') (tensor(140.7906, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.8966 (0.8379791281440041) + | > loader_time: 0.0121 (0.011437839811498469) + + + --> TIME: 2024-06-25 07:45:34 -- STEP: 245/591 -- GLOBAL_STEP: 17975 + | > loss_disc: 2.677731990814209 (2.725735619603372) + | > loss_disc_real_0: 0.21233543753623962 (0.21088428287481775) + | > loss_disc_real_1: 0.23888856172561646 (0.22929503145266553) + | > loss_disc_real_2: 0.2064162641763687 (0.23589925145616336) + | > loss_disc_real_3: 0.23643141984939575 (0.23559448864995217) + | > loss_disc_real_4: 0.23220063745975494 (0.23337224381310598) + | > loss_disc_real_5: 0.24304628372192383 (0.2385848508197434) + | > loss_0: 2.677731990814209 (2.725735619603372) + | > grad_norm_0: tensor(5.9424, device='cuda:0') (tensor(10.1099, device='cuda:0')) + | > loss_gen: 1.8852508068084717 (1.9115368341913028) + | > loss_kl: 2.5326013565063477 (2.5378021249965745) + | > loss_feat: 3.7150933742523193 (3.6088920826814608) + | > loss_mel: 20.973480224609375 (21.1590431213379) + | > loss_duration: 1.3678442239761353 (1.3556126614006203) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.47426986694336 (30.57288682120187) + | > grad_norm_1: tensor(74.6545, device='cuda:0') (tensor(137.3048, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.8833 (0.8431123305340202) + | > loader_time: 0.0129 (0.011725089987929987) + + + --> TIME: 2024-06-25 07:45:57 -- STEP: 270/591 -- GLOBAL_STEP: 18000 + | > loss_disc: 2.6913578510284424 (2.727043709048519) + | > loss_disc_real_0: 0.1493043750524521 (0.21170303691868428) + | > loss_disc_real_1: 0.22366493940353394 (0.2287832432874927) + | > loss_disc_real_2: 0.2397402673959732 (0.2357971186991091) + | > loss_disc_real_3: 0.24217072129249573 (0.23552701191769707) + | > loss_disc_real_4: 0.22950506210327148 (0.2333173350603492) + | > loss_disc_real_5: 0.2599530518054962 (0.23866215061258386) + | > loss_0: 2.6913578510284424 (2.727043709048519) + | > grad_norm_0: tensor(10.6174, device='cuda:0') (tensor(10.4900, device='cuda:0')) + | > loss_gen: 1.8590166568756104 (1.9102838432347333) + | > loss_kl: 2.4172322750091553 (2.5381825535385696) + | > loss_feat: 3.7167937755584717 (3.60073237242522) + | > loss_mel: 21.31851577758789 (21.132599300808383) + | > loss_duration: 1.3726253509521484 (1.3554934607611766) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.68418312072754 (30.537291512665924) + | > grad_norm_1: tensor(65.6540, device='cuda:0') (tensor(134.1503, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.8801 (0.8495499557918972) + | > loader_time: 0.0168 (0.012010683836760344) + + + --> TIME: 2024-06-25 07:46:20 -- STEP: 295/591 -- GLOBAL_STEP: 18025 + | > loss_disc: 2.6280267238616943 (2.7263569912668015) + | > loss_disc_real_0: 0.2103937417268753 (0.2116955401281179) + | > loss_disc_real_1: 0.2252148687839508 (0.2286301734588914) + | > loss_disc_real_2: 0.2275705486536026 (0.2357429597337367) + | > loss_disc_real_3: 0.2196577936410904 (0.23549354051129293) + | > loss_disc_real_4: 0.1862994134426117 (0.23315577744427374) + | > loss_disc_real_5: 0.19799776375293732 (0.23858195174548585) + | > loss_0: 2.6280267238616943 (2.7263569912668015) + | > grad_norm_0: tensor(4.5705, device='cuda:0') (tensor(10.2018, device='cuda:0')) + | > loss_gen: 1.8310397863388062 (1.9079846911511178) + | > loss_kl: 2.4327831268310547 (2.5349120746224623) + | > loss_feat: 3.650456666946411 (3.594111003714091) + | > loss_mel: 21.496049880981445 (21.122505336696822) + | > loss_duration: 1.3487204313278198 (1.3564160795535076) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.759048461914062 (30.515929193011786) + | > grad_norm_1: tensor(287.8501, device='cuda:0') (tensor(133.8118, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.9097 (0.8546947883347334) + | > loader_time: 0.0144 (0.012282294742131637) + + + --> TIME: 2024-06-25 07:46:44 -- STEP: 320/591 -- GLOBAL_STEP: 18050 + | > loss_disc: 2.702152729034424 (2.7256372816860694) + | > loss_disc_real_0: 0.18794628977775574 (0.2115498006576672) + | > loss_disc_real_1: 0.19979941844940186 (0.22843479118309915) + | > loss_disc_real_2: 0.2313186079263687 (0.23552633253857494) + | > loss_disc_real_3: 0.26999446749687195 (0.2355860907584429) + | > loss_disc_real_4: 0.2091255635023117 (0.23344253483228386) + | > loss_disc_real_5: 0.22401423752307892 (0.23854021434672176) + | > loss_0: 2.702152729034424 (2.7256372816860694) + | > grad_norm_0: tensor(10.5063, device='cuda:0') (tensor(10.1696, device='cuda:0')) + | > loss_gen: 1.9211339950561523 (1.9079593539237976) + | > loss_kl: 2.5672614574432373 (2.5347070321440697) + | > loss_feat: 3.6673710346221924 (3.593127308785914) + | > loss_mel: 21.81842613220215 (21.133350950479503) + | > loss_duration: 1.3406753540039062 (1.3572352204471831) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.314868927001953 (30.526379865407943) + | > grad_norm_1: tensor(171.9638, device='cuda:0') (tensor(135.2086, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.9253 (0.8599703297019005) + | > loader_time: 0.0155 (0.012514631450176238) + + + --> TIME: 2024-06-25 07:47:08 -- STEP: 345/591 -- GLOBAL_STEP: 18075 + | > loss_disc: 2.7774581909179688 (2.7253385460895063) + | > loss_disc_real_0: 0.21411941945552826 (0.21138022077688273) + | > loss_disc_real_1: 0.23176702857017517 (0.22825635192186935) + | > loss_disc_real_2: 0.1910669505596161 (0.23556085272111754) + | > loss_disc_real_3: 0.24902786314487457 (0.23555500904719034) + | > loss_disc_real_4: 0.21533915400505066 (0.2334277298139489) + | > loss_disc_real_5: 0.20837447047233582 (0.23840144116809403) + | > loss_0: 2.7774581909179688 (2.7253385460895063) + | > grad_norm_0: tensor(3.7257, device='cuda:0') (tensor(10.1659, device='cuda:0')) + | > loss_gen: 1.969208836555481 (1.9074999850729237) + | > loss_kl: 2.6018757820129395 (2.531768893504488) + | > loss_feat: 3.396737813949585 (3.590294298227281) + | > loss_mel: 20.107383728027344 (21.116720818782202) + | > loss_duration: 1.408639669418335 (1.358445835113526) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.48384666442871 (30.504729832082553) + | > grad_norm_1: tensor(47.5723, device='cuda:0') (tensor(132.1655, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.9557 (0.865026583187822) + | > loader_time: 0.0141 (0.012748459111089292) + + + --> TIME: 2024-06-25 07:47:32 -- STEP: 370/591 -- GLOBAL_STEP: 18100 + | > loss_disc: 2.7392444610595703 (2.7258254528045653) + | > loss_disc_real_0: 0.22131803631782532 (0.21169725040326248) + | > loss_disc_real_1: 0.22416427731513977 (0.22813028781800657) + | > loss_disc_real_2: 0.21416105329990387 (0.23584466008721172) + | > loss_disc_real_3: 0.18815769255161285 (0.23552428818232304) + | > loss_disc_real_4: 0.19802364706993103 (0.23332592184479173) + | > loss_disc_real_5: 0.2334323674440384 (0.23839713721661954) + | > loss_0: 2.7392444610595703 (2.7258254528045653) + | > grad_norm_0: tensor(7.9484, device='cuda:0') (tensor(10.2149, device='cuda:0')) + | > loss_gen: 1.8771320581436157 (1.9064016081191397) + | > loss_kl: 2.559588670730591 (2.530259384335699) + | > loss_feat: 3.6219356060028076 (3.581452273678134) + | > loss_mel: 21.084510803222656 (21.109632925085116) + | > loss_duration: 1.3631879091262817 (1.3598778998529593) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.50635528564453 (30.487624085916057) + | > grad_norm_1: tensor(191.7785, device='cuda:0') (tensor(135.8514, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.9429 (0.8702401077425158) + | > loader_time: 0.0152 (0.013010282129854769) + + + --> TIME: 2024-06-25 07:47:56 -- STEP: 395/591 -- GLOBAL_STEP: 18125 + | > loss_disc: 2.7349443435668945 (2.726279888877386) + | > loss_disc_real_0: 0.2405281960964203 (0.21182516675583923) + | > loss_disc_real_1: 0.20920008420944214 (0.22813625633716583) + | > loss_disc_real_2: 0.24937890470027924 (0.23600502685655522) + | > loss_disc_real_3: 0.2016599476337433 (0.23559462345099147) + | > loss_disc_real_4: 0.22944046556949615 (0.23346791463562205) + | > loss_disc_real_5: 0.25963494181632996 (0.23867178676249107) + | > loss_0: 2.7349443435668945 (2.726279888877386) + | > grad_norm_0: tensor(8.0946, device='cuda:0') (tensor(10.3982, device='cuda:0')) + | > loss_gen: 1.7408267259597778 (1.9059002483947367) + | > loss_kl: 2.56095290184021 (2.5297629169270968) + | > loss_feat: 3.2338409423828125 (3.574115295048) + | > loss_mel: 21.35775375366211 (21.098949132991727) + | > loss_duration: 1.380777359008789 (1.360569730589662) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.274152755737305 (30.46929730282554) + | > grad_norm_1: tensor(210.9048, device='cuda:0') (tensor(138.0203, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.9572 (0.8761984318117552) + | > loader_time: 0.0161 (0.013279128980033006) + + + --> TIME: 2024-06-25 07:48:21 -- STEP: 420/591 -- GLOBAL_STEP: 18150 + | > loss_disc: 2.6775870323181152 (2.725382968357631) + | > loss_disc_real_0: 0.17559757828712463 (0.2112872724909158) + | > loss_disc_real_1: 0.2125881314277649 (0.2281163701344104) + | > loss_disc_real_2: 0.20127706229686737 (0.2359824888408184) + | > loss_disc_real_3: 0.23354333639144897 (0.23558258190751075) + | > loss_disc_real_4: 0.2608337104320526 (0.2335509602512632) + | > loss_disc_real_5: 0.2737399935722351 (0.23858492651156016) + | > loss_0: 2.6775870323181152 (2.725382968357631) + | > grad_norm_0: tensor(5.5255, device='cuda:0') (tensor(10.2569, device='cuda:0')) + | > loss_gen: 1.8730509281158447 (1.905071701322283) + | > loss_kl: 2.570035934448242 (2.5270851135253904) + | > loss_feat: 3.4205689430236816 (3.567729257401965) + | > loss_mel: 21.12255096435547 (21.088059543427953) + | > loss_duration: 1.3844082355499268 (1.36137789175624) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.370615005493164 (30.44932347706386) + | > grad_norm_1: tensor(106.1690, device='cuda:0') (tensor(138.3232, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.9864 (0.8823413905643281) + | > loader_time: 0.0201 (0.013548202741713752) + + + --> TIME: 2024-06-25 07:48:47 -- STEP: 445/591 -- GLOBAL_STEP: 18175 + | > loss_disc: 2.710559844970703 (2.725144266814328) + | > loss_disc_real_0: 0.18539205193519592 (0.21122845548592256) + | > loss_disc_real_1: 0.25682052969932556 (0.22808760207020834) + | > loss_disc_real_2: 0.2323862761259079 (0.23581742332892472) + | > loss_disc_real_3: 0.2547597587108612 (0.23576067471772097) + | > loss_disc_real_4: 0.25135934352874756 (0.2334575214412775) + | > loss_disc_real_5: 0.2597143054008484 (0.23854587637306598) + | > loss_0: 2.710559844970703 (2.725144266814328) + | > grad_norm_0: tensor(6.7856, device='cuda:0') (tensor(10.2791, device='cuda:0')) + | > loss_gen: 1.909218430519104 (1.9048252277159958) + | > loss_kl: 2.4460537433624268 (2.5249000313576686) + | > loss_feat: 3.3883495330810547 (3.5611855281872686) + | > loss_mel: 21.02372932434082 (21.07678189652688) + | > loss_duration: 1.4109399318695068 (1.3627188130710908) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.17828941345215 (30.430411478106894) + | > grad_norm_1: tensor(169.6298, device='cuda:0') (tensor(137.8168, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 1.0138 (0.8892493285489886) + | > loader_time: 0.0176 (0.013833155792750669) + + + --> TIME: 2024-06-25 07:49:13 -- STEP: 470/591 -- GLOBAL_STEP: 18200 + | > loss_disc: 2.7511019706726074 (2.727244146326755) + | > loss_disc_real_0: 0.12389737367630005 (0.21153403748857214) + | > loss_disc_real_1: 0.23199313879013062 (0.2281022198339726) + | > loss_disc_real_2: 0.2310916781425476 (0.23627289460694537) + | > loss_disc_real_3: 0.19306884706020355 (0.23580813141579324) + | > loss_disc_real_4: 0.19779571890830994 (0.23349622212191845) + | > loss_disc_real_5: 0.19228656589984894 (0.23852167081959705) + | > loss_0: 2.7511019706726074 (2.727244146326755) + | > grad_norm_0: tensor(18.5594, device='cuda:0') (tensor(10.4594, device='cuda:0')) + | > loss_gen: 1.787590742111206 (1.902861956839866) + | > loss_kl: 2.3719115257263184 (2.5223217451826057) + | > loss_feat: 3.264986276626587 (3.551424414046266) + | > loss_mel: 20.42906951904297 (21.059418406385046) + | > loss_duration: 1.41412353515625 (1.3638073119711374) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.267681121826172 (30.399833829352197) + | > grad_norm_1: tensor(60.6570, device='cuda:0') (tensor(135.7737, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 0.9949 (0.8962622328007475) + | > loader_time: 0.0182 (0.014116038160121188) + + + --> TIME: 2024-06-25 07:49:40 -- STEP: 495/591 -- GLOBAL_STEP: 18225 + | > loss_disc: 2.7596282958984375 (2.727609585270737) + | > loss_disc_real_0: 0.12126556783914566 (0.21145798261418486) + | > loss_disc_real_1: 0.226057767868042 (0.2279890927401456) + | > loss_disc_real_2: 0.21381522715091705 (0.23657408690813816) + | > loss_disc_real_3: 0.22404325008392334 (0.23582806231999637) + | > loss_disc_real_4: 0.23216399550437927 (0.2336368550254841) + | > loss_disc_real_5: 0.23341038823127747 (0.23856057316967935) + | > loss_0: 2.7596282958984375 (2.727609585270737) + | > grad_norm_0: tensor(26.1934, device='cuda:0') (tensor(10.3366, device='cuda:0')) + | > loss_gen: 2.0678811073303223 (1.9030160636612863) + | > loss_kl: 2.496303081512451 (2.5214994917012223) + | > loss_feat: 3.547724723815918 (3.544855271927033) + | > loss_mel: 20.598026275634766 (21.042003211589762) + | > loss_duration: 1.3911126852035522 (1.3650280954861886) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.10104751586914 (30.37640213629212) + | > grad_norm_1: tensor(37.5329, device='cuda:0') (tensor(134.8562, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 1.0847 (0.9041110679356739) + | > loader_time: 0.0221 (0.014444554454148419) + + + --> TIME: 2024-06-25 07:50:09 -- STEP: 520/591 -- GLOBAL_STEP: 18250 + | > loss_disc: 2.79622745513916 (2.72888146180373) + | > loss_disc_real_0: 0.13160842657089233 (0.21154939401894807) + | > loss_disc_real_1: 0.23697715997695923 (0.2283927692243686) + | > loss_disc_real_2: 0.25099000334739685 (0.23654780651514346) + | > loss_disc_real_3: 0.2868540287017822 (0.2358459354020082) + | > loss_disc_real_4: 0.24695464968681335 (0.23361449470886816) + | > loss_disc_real_5: 0.2611062526702881 (0.23873416552176843) + | > loss_0: 2.79622745513916 (2.72888146180373) + | > grad_norm_0: tensor(20.8623, device='cuda:0') (tensor(10.3659, device='cuda:0')) + | > loss_gen: 1.9442832469940186 (1.9013743130060343) + | > loss_kl: 2.4768409729003906 (2.519667727672137) + | > loss_feat: 2.9981276988983154 (3.5346826649629146) + | > loss_mel: 20.18735694885254 (21.02532296547522) + | > loss_duration: 1.3950307369232178 (1.3666304398041509) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.00164031982422 (30.347678096477804) + | > grad_norm_1: tensor(237.2803, device='cuda:0') (tensor(134.0631, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 1.2348 (0.9138853577467111) + | > loader_time: 0.0246 (0.014760138438298152) + + + --> TIME: 2024-06-25 07:50:39 -- STEP: 545/591 -- GLOBAL_STEP: 18275 + | > loss_disc: 2.685119152069092 (2.7294771570678145) + | > loss_disc_real_0: 0.28084853291511536 (0.21163764878970767) + | > loss_disc_real_1: 0.18884673714637756 (0.2283164643639818) + | > loss_disc_real_2: 0.2577095031738281 (0.23686300056789994) + | > loss_disc_real_3: 0.1934114396572113 (0.2357778732656339) + | > loss_disc_real_4: 0.2173660844564438 (0.23367736388783936) + | > loss_disc_real_5: 0.2355809509754181 (0.23869968048476298) + | > loss_0: 2.685119152069092 (2.7294771570678145) + | > grad_norm_0: tensor(15.7230, device='cuda:0') (tensor(10.4062, device='cuda:0')) + | > loss_gen: 1.8040138483047485 (1.900415171395748) + | > loss_kl: 2.481743574142456 (2.518242437030197) + | > loss_feat: 3.4573609828948975 (3.5264782839958815) + | > loss_mel: 21.225826263427734 (21.013557507576188) + | > loss_duration: 1.3766238689422607 (1.3682468514923658) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.34556770324707 (30.326940246022076) + | > grad_norm_1: tensor(203.1134, device='cuda:0') (tensor(136.3164, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 1.2489 (0.9259218968382668) + | > loader_time: 0.0242 (0.015088940541678612) + + + --> TIME: 2024-06-25 07:51:10 -- STEP: 570/591 -- GLOBAL_STEP: 18300 + | > loss_disc: 2.7115838527679443 (2.729632281420523) + | > loss_disc_real_0: 0.17843425273895264 (0.21153624022477552) + | > loss_disc_real_1: 0.22124534845352173 (0.2282797459709017) + | > loss_disc_real_2: 0.2386825531721115 (0.23694098793100893) + | > loss_disc_real_3: 0.22995737195014954 (0.2358073307495368) + | > loss_disc_real_4: 0.2237403392791748 (0.23358522752920788) + | > loss_disc_real_5: 0.24134288728237152 (0.23868904712430217) + | > loss_0: 2.7115838527679443 (2.729632281420523) + | > grad_norm_0: tensor(7.2032, device='cuda:0') (tensor(10.3142, device='cuda:0')) + | > loss_gen: 2.00230073928833 (1.8996081197470949) + | > loss_kl: 2.4747047424316406 (2.5163662822623) + | > loss_feat: 3.4297711849212646 (3.519248412366499) + | > loss_mel: 21.286813735961914 (20.996670619228418) + | > loss_duration: 1.3966479301452637 (1.3697943377913093) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.590238571166992 (30.30168776261179) + | > grad_norm_1: tensor(244.5803, device='cuda:0') (tensor(135.3251, device='cuda:0')) + | > current_lr_0: 0.00019925135779039958 + | > current_lr_1: 0.00019925135779039958 + | > step_time: 1.2241 (0.9392318629381945) + | > loader_time: 0.0223 (0.015453401364778218) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037375561734463306 (+0.00015668158835553106) + | > avg_loss_disc: 2.7317268341145615 (+0.01625560699625206) + | > avg_loss_disc_real_0: 0.285958746963359 (+0.11786204085071042) + | > avg_loss_disc_real_1: 0.25267930075209194 (+0.05242200798176708) + | > avg_loss_disc_real_2: 0.23377891170217635 (-0.011627611327678616) + | > avg_loss_disc_real_3: 0.23843168102680368 (+0.05324539383675189) + | > avg_loss_disc_real_4: 0.2566436583057363 (+0.0177502381674787) + | > avg_loss_disc_real_5: 0.24602892582720898 (+0.007484245807566542) + | > avg_loss_0: 2.7317268341145615 (+0.01625560699625206) + | > avg_loss_gen: 2.042875622181183 (+0.2961107264173799) + | > avg_loss_kl: 2.5980856672246406 (+0.024821108960090488) + | > avg_loss_feat: 3.4468934941799083 (-0.006222212568242291) + | > avg_loss_mel: 20.512577259794188 (-0.33384018755974054) + | > avg_loss_duration: 1.3745544235757057 (-0.034823727100454116) + | > avg_loss_1: 29.97498654304667 (-0.053954022995966966) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_18321.pth + + > EPOCH: 32/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 07:51:50)  + + --> TIME: 2024-06-25 07:51:55 -- STEP: 4/591 -- GLOBAL_STEP: 18325 + | > loss_disc: 2.717505931854248 (2.7022310495376587) + | > loss_disc_real_0: 0.16176965832710266 (0.18747609108686447) + | > loss_disc_real_1: 0.2322368323802948 (0.23784074932336807) + | > loss_disc_real_2: 0.24564334750175476 (0.2336624674499035) + | > loss_disc_real_3: 0.23104950785636902 (0.231938898563385) + | > loss_disc_real_4: 0.1999480426311493 (0.229736328125) + | > loss_disc_real_5: 0.2088659405708313 (0.24920855835080147) + | > loss_0: 2.717505931854248 (2.7022310495376587) + | > grad_norm_0: tensor(7.1862, device='cuda:0') (tensor(8.0842, device='cuda:0')) + | > loss_gen: 1.9181138277053833 (1.9082632958889008) + | > loss_kl: 2.6923065185546875 (2.7152139544487) + | > loss_feat: 3.935980796813965 (3.8592111468315125) + | > loss_mel: 21.501220703125 (21.17998695373535) + | > loss_duration: 1.3594911098480225 (1.3615642786026) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.407114028930664 (31.024239540100098) + | > grad_norm_1: tensor(124.6560, device='cuda:0') (tensor(155.1067, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.7714 (0.7747081518173218) + | > loader_time: 0.0093 (0.008261620998382568) + + + --> TIME: 2024-06-25 07:52:15 -- STEP: 29/591 -- GLOBAL_STEP: 18350 + | > loss_disc: 2.7345433235168457 (2.7149256262285952) + | > loss_disc_real_0: 0.15044331550598145 (0.20652547118992642) + | > loss_disc_real_1: 0.23772463202476501 (0.225689645471244) + | > loss_disc_real_2: 0.25895199179649353 (0.23794743418693542) + | > loss_disc_real_3: 0.27140873670578003 (0.2374235905449966) + | > loss_disc_real_4: 0.22045190632343292 (0.2339153361731562) + | > loss_disc_real_5: 0.23708897829055786 (0.2403708978973586) + | > loss_0: 2.7345433235168457 (2.7149256262285952) + | > grad_norm_0: tensor(12.3908, device='cuda:0') (tensor(11.1464, device='cuda:0')) + | > loss_gen: 1.8498867750167847 (1.931466480781292) + | > loss_kl: 2.636681318283081 (2.643266595643142) + | > loss_feat: 3.5130953788757324 (3.758248551138516) + | > loss_mel: 20.32993507385254 (21.04671234920107) + | > loss_duration: 1.350439190864563 (1.331137936690758) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.680036544799805 (30.710831806577485) + | > grad_norm_1: tensor(239.3794, device='cuda:0') (tensor(153.0160, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.7883 (0.7843640097256365) + | > loader_time: 0.0089 (0.009014910665051692) + + + --> TIME: 2024-06-25 07:52:36 -- STEP: 54/591 -- GLOBAL_STEP: 18375 + | > loss_disc: 2.664391279220581 (2.7203654094978615) + | > loss_disc_real_0: 0.1653306633234024 (0.21010170545842913) + | > loss_disc_real_1: 0.22903147339820862 (0.22565358187313433) + | > loss_disc_real_2: 0.25369393825531006 (0.2370094229225759) + | > loss_disc_real_3: 0.2510480582714081 (0.23472972904090528) + | > loss_disc_real_4: 0.19915327429771423 (0.2348998092390873) + | > loss_disc_real_5: 0.21407832205295563 (0.23962511784500545) + | > loss_0: 2.664391279220581 (2.7203654094978615) + | > grad_norm_0: tensor(8.5163, device='cuda:0') (tensor(11.0991, device='cuda:0')) + | > loss_gen: 1.9434773921966553 (1.917318375022323) + | > loss_kl: 2.545510768890381 (2.612987023812753) + | > loss_feat: 3.901761293411255 (3.7005459600024753) + | > loss_mel: 21.264638900756836 (21.053986725983798) + | > loss_duration: 1.3198291063308716 (1.3427333059134308) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.975217819213867 (30.627571353205926) + | > grad_norm_1: tensor(185.7385, device='cuda:0') (tensor(157.5844, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.8314 (0.7931169315620704) + | > loader_time: 0.01 (0.009572285192984123) + + + --> TIME: 2024-06-25 07:52:56 -- STEP: 79/591 -- GLOBAL_STEP: 18400 + | > loss_disc: 2.711664915084839 (2.722753874863251) + | > loss_disc_real_0: 0.2740596830844879 (0.21053515412384952) + | > loss_disc_real_1: 0.27326449751853943 (0.22758249429207814) + | > loss_disc_real_2: 0.23512880504131317 (0.23617938896523247) + | > loss_disc_real_3: 0.2596665322780609 (0.23501315675204312) + | > loss_disc_real_4: 0.20220503211021423 (0.23413425816010824) + | > loss_disc_real_5: 0.2129247784614563 (0.23869644935372508) + | > loss_0: 2.711664915084839 (2.722753874863251) + | > grad_norm_0: tensor(9.6899, device='cuda:0') (tensor(10.6758, device='cuda:0')) + | > loss_gen: 1.9369537830352783 (1.9151698139649402) + | > loss_kl: 2.5762031078338623 (2.594004760814618) + | > loss_feat: 3.787888288497925 (3.6803870563265644) + | > loss_mel: 21.43734359741211 (21.08521306967434) + | > loss_duration: 1.331127405166626 (1.3502131821234011) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.069517135620117 (30.624987855742248) + | > grad_norm_1: tensor(45.5590, device='cuda:0') (tensor(139.4825, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.8241 (0.8018772632260865) + | > loader_time: 0.0106 (0.010086814059486873) + + + --> TIME: 2024-06-25 07:53:18 -- STEP: 104/591 -- GLOBAL_STEP: 18425 + | > loss_disc: 2.705718994140625 (2.721396785516006) + | > loss_disc_real_0: 0.1953580677509308 (0.2094879328058316) + | > loss_disc_real_1: 0.2144070714712143 (0.22726277276300466) + | > loss_disc_real_2: 0.23175249993801117 (0.23572589729267818) + | > loss_disc_real_3: 0.21168413758277893 (0.23524942549948508) + | > loss_disc_real_4: 0.21939770877361298 (0.2344720091384191) + | > loss_disc_real_5: 0.2606935203075409 (0.2397220151928755) + | > loss_0: 2.705718994140625 (2.721396785516006) + | > grad_norm_0: tensor(8.0693, device='cuda:0') (tensor(10.2142, device='cuda:0')) + | > loss_gen: 1.936142921447754 (1.9118961966954744) + | > loss_kl: 2.5190541744232178 (2.5745356082916255) + | > loss_feat: 3.5605006217956543 (3.6687528376395884) + | > loss_mel: 20.591812133789062 (21.092142838698177) + | > loss_duration: 1.3385522365570068 (1.3563992461332908) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.946062088012695 (30.60372671714196) + | > grad_norm_1: tensor(212.8522, device='cuda:0') (tensor(148.6462, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.8293 (0.8093078824189993) + | > loader_time: 0.0117 (0.010354305689151473) + + + --> TIME: 2024-06-25 07:53:39 -- STEP: 129/591 -- GLOBAL_STEP: 18450 + | > loss_disc: 2.742793321609497 (2.723188243171042) + | > loss_disc_real_0: 0.2119179517030716 (0.2104910669862762) + | > loss_disc_real_1: 0.22030778229236603 (0.22741531736628953) + | > loss_disc_real_2: 0.26635420322418213 (0.23592383443385131) + | > loss_disc_real_3: 0.2096862494945526 (0.23537695038226225) + | > loss_disc_real_4: 0.2501445412635803 (0.23428732849830805) + | > loss_disc_real_5: 0.26516562700271606 (0.23931765579438025) + | > loss_0: 2.742793321609497 (2.723188243171042) + | > grad_norm_0: tensor(3.5800, device='cuda:0') (tensor(10.3173, device='cuda:0')) + | > loss_gen: 1.8616347312927246 (1.9106397906015085) + | > loss_kl: 2.5133841037750244 (2.5620676879734954) + | > loss_feat: 3.4905364513397217 (3.6626960029897764) + | > loss_mel: 20.731380462646484 (21.0878327244012) + | > loss_duration: 1.3346762657165527 (1.35761035412781) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.93161392211914 (30.580846535143003) + | > grad_norm_1: tensor(43.6892, device='cuda:0') (tensor(143.4404, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.8663 (0.8156578651694364) + | > loader_time: 0.0122 (0.01057447204294131) + + + --> TIME: 2024-06-25 07:54:01 -- STEP: 154/591 -- GLOBAL_STEP: 18475 + | > loss_disc: 2.7110612392425537 (2.720901656460452) + | > loss_disc_real_0: 0.17365586757659912 (0.2095590564724687) + | > loss_disc_real_1: 0.24599042534828186 (0.22765833071687006) + | > loss_disc_real_2: 0.24150429666042328 (0.23592102188955655) + | > loss_disc_real_3: 0.23869144916534424 (0.2354157059997707) + | > loss_disc_real_4: 0.22091886401176453 (0.2339155308224938) + | > loss_disc_real_5: 0.240584135055542 (0.23908313441199142) + | > loss_0: 2.7110612392425537 (2.720901656460452) + | > grad_norm_0: tensor(7.2816, device='cuda:0') (tensor(10.1256, device='cuda:0')) + | > loss_gen: 1.8133726119995117 (1.9120257187199283) + | > loss_kl: 2.3652079105377197 (2.554517834217517) + | > loss_feat: 3.6021337509155273 (3.6679154154542206) + | > loss_mel: 21.072574615478516 (21.10271427848122) + | > loss_duration: 1.332096815109253 (1.3561781358409235) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.185386657714844 (30.59335135175036) + | > grad_norm_1: tensor(62.3627, device='cuda:0') (tensor(138.9902, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.8458 (0.8219852865516366) + | > loader_time: 0.0117 (0.010843818837946113) + + + --> TIME: 2024-06-25 07:54:23 -- STEP: 179/591 -- GLOBAL_STEP: 18500 + | > loss_disc: 2.778712749481201 (2.7242412047679183) + | > loss_disc_real_0: 0.19610720872879028 (0.21041940027775044) + | > loss_disc_real_1: 0.2476700097322464 (0.22785299258858133) + | > loss_disc_real_2: 0.23914210498332977 (0.23646767237665933) + | > loss_disc_real_3: 0.22996532917022705 (0.23526769518519247) + | > loss_disc_real_4: 0.23218853771686554 (0.23400365706928616) + | > loss_disc_real_5: 0.24984049797058105 (0.23931895562723363) + | > loss_0: 2.778712749481201 (2.7242412047679183) + | > grad_norm_0: tensor(4.0782, device='cuda:0') (tensor(10.4863, device='cuda:0')) + | > loss_gen: 1.8828709125518799 (1.911162020773861) + | > loss_kl: 2.504739284515381 (2.5500107610691862) + | > loss_feat: 3.7110629081726074 (3.6574942945768045) + | > loss_mel: 20.71780014038086 (21.082502716746394) + | > loss_duration: 1.3829697370529175 (1.3551476507879503) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.19944190979004 (30.55631741997916) + | > grad_norm_1: tensor(94.1418, device='cuda:0') (tensor(133.0150, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.8635 (0.8274410863162419) + | > loader_time: 0.0116 (0.010953082718662705) + + + --> TIME: 2024-06-25 07:54:45 -- STEP: 204/591 -- GLOBAL_STEP: 18525 + | > loss_disc: 2.7086336612701416 (2.7222322073637257) + | > loss_disc_real_0: 0.13037464022636414 (0.21020615838614165) + | > loss_disc_real_1: 0.23135459423065186 (0.22746503820606306) + | > loss_disc_real_2: 0.23541349172592163 (0.23672004862158907) + | > loss_disc_real_3: 0.19730868935585022 (0.23495352268218994) + | > loss_disc_real_4: 0.21939098834991455 (0.2336381024443636) + | > loss_disc_real_5: 0.2383386194705963 (0.2392070727313266) + | > loss_0: 2.7086336612701416 (2.7222322073637257) + | > grad_norm_0: tensor(20.7801, device='cuda:0') (tensor(10.1995, device='cuda:0')) + | > loss_gen: 1.9824609756469727 (1.9121492475855584) + | > loss_kl: 2.3817648887634277 (2.542908140257293) + | > loss_feat: 3.7120704650878906 (3.6619704435853397) + | > loss_mel: 21.182401657104492 (21.09771677092008) + | > loss_duration: 1.3295214176177979 (1.3544506261161733) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.588218688964844 (30.56919521443984) + | > grad_norm_1: tensor(82.6119, device='cuda:0') (tensor(131.4655, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.8685 (0.8326934237106174) + | > loader_time: 0.0119 (0.011137722754010966) + + + --> TIME: 2024-06-25 07:55:08 -- STEP: 229/591 -- GLOBAL_STEP: 18550 + | > loss_disc: 2.7985708713531494 (2.724490715426648) + | > loss_disc_real_0: 0.16271810233592987 (0.21128696030794794) + | > loss_disc_real_1: 0.21944929659366608 (0.22780185436838057) + | > loss_disc_real_2: 0.25903624296188354 (0.23685398587233114) + | > loss_disc_real_3: 0.2359132468700409 (0.23516333857998578) + | > loss_disc_real_4: 0.25176915526390076 (0.23316449629687847) + | > loss_disc_real_5: 0.24945484101772308 (0.23888909862291344) + | > loss_0: 2.7985708713531494 (2.724490715426648) + | > grad_norm_0: tensor(13.2469, device='cuda:0') (tensor(10.3293, device='cuda:0')) + | > loss_gen: 1.9486839771270752 (1.9107734214791043) + | > loss_kl: 2.540517568588257 (2.538360538441021) + | > loss_feat: 3.4606473445892334 (3.6482233209901502) + | > loss_mel: 21.57199478149414 (21.108708194249566) + | > loss_duration: 1.3987191915512085 (1.3541841762034657) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.920564651489258 (30.560249636787514) + | > grad_norm_1: tensor(136.5022, device='cuda:0') (tensor(128.3383, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.8855 (0.8378470693613244) + | > loader_time: 0.0139 (0.011444875767137284) + + + --> TIME: 2024-06-25 07:55:31 -- STEP: 254/591 -- GLOBAL_STEP: 18575 + | > loss_disc: 2.7016284465789795 (2.72454011440277) + | > loss_disc_real_0: 0.13050176203250885 (0.2112579128932296) + | > loss_disc_real_1: 0.19434525072574615 (0.22779338507671057) + | > loss_disc_real_2: 0.2362154722213745 (0.23695172005751003) + | > loss_disc_real_3: 0.25254082679748535 (0.23527672963114235) + | > loss_disc_real_4: 0.2365308552980423 (0.23294383680491937) + | > loss_disc_real_5: 0.24712835252285004 (0.23906243287437545) + | > loss_0: 2.7016284465789795 (2.72454011440277) + | > grad_norm_0: tensor(16.3059, device='cuda:0') (tensor(10.2653, device='cuda:0')) + | > loss_gen: 1.8427008390426636 (1.9087229319444792) + | > loss_kl: 2.4934470653533936 (2.5341217104844223) + | > loss_feat: 3.561793565750122 (3.6423959825921246) + | > loss_mel: 20.728986740112305 (21.103920621196103) + | > loss_duration: 1.322771668434143 (1.3541747608522732) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.94969940185547 (30.54333597468579) + | > grad_norm_1: tensor(71.0481, device='cuda:0') (tensor(126.3383, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.9099 (0.8440747110862431) + | > loader_time: 0.0149 (0.011756253993417332) + + + --> TIME: 2024-06-25 07:55:54 -- STEP: 279/591 -- GLOBAL_STEP: 18600 + | > loss_disc: 2.700329065322876 (2.7226397606634314) + | > loss_disc_real_0: 0.23444625735282898 (0.21103831434014875) + | > loss_disc_real_1: 0.21133509278297424 (0.22735851068436888) + | > loss_disc_real_2: 0.2556367814540863 (0.23690883580288152) + | > loss_disc_real_3: 0.30094513297080994 (0.23528684798534627) + | > loss_disc_real_4: 0.22820475697517395 (0.23282790605953518) + | > loss_disc_real_5: 0.22368526458740234 (0.23903190502129149) + | > loss_0: 2.700329065322876 (2.7226397606634314) + | > grad_norm_0: tensor(8.7934, device='cuda:0') (tensor(9.9769, device='cuda:0')) + | > loss_gen: 1.9387187957763672 (1.9095106069332382) + | > loss_kl: 2.5169591903686523 (2.529482245017978) + | > loss_feat: 3.6257951259613037 (3.6449596258047228) + | > loss_mel: 21.297632217407227 (21.09298922084137) + | > loss_duration: 1.3502827882766724 (1.3546130276922685) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.729389190673828 (30.53155470407137) + | > grad_norm_1: tensor(202.7021, device='cuda:0') (tensor(126.8640, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.9123 (0.8492772715920616) + | > loader_time: 0.0159 (0.012052756483836838) + + + --> TIME: 2024-06-25 07:56:17 -- STEP: 304/591 -- GLOBAL_STEP: 18625 + | > loss_disc: 2.786151885986328 (2.724325962756809) + | > loss_disc_real_0: 0.13943764567375183 (0.2109624274859303) + | > loss_disc_real_1: 0.24278992414474487 (0.22747861216530987) + | > loss_disc_real_2: 0.2372383326292038 (0.23694611027052528) + | > loss_disc_real_3: 0.28278812766075134 (0.23547766259626338) + | > loss_disc_real_4: 0.21804149448871613 (0.23277346936887816) + | > loss_disc_real_5: 0.21791422367095947 (0.23904062033091722) + | > loss_0: 2.786151885986328 (2.724325962756809) + | > grad_norm_0: tensor(14.2050, device='cuda:0') (tensor(10.1265, device='cuda:0')) + | > loss_gen: 2.078964948654175 (1.9085888168529461) + | > loss_kl: 2.577207088470459 (2.528362878843357) + | > loss_feat: 3.6342368125915527 (3.6395914468326067) + | > loss_mel: 21.493268966674805 (21.082076637368434) + | > loss_duration: 1.3989205360412598 (1.3553159864325277) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.18259620666504 (30.513935747899506) + | > grad_norm_1: tensor(46.6977, device='cuda:0') (tensor(123.8765, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.9129 (0.8545975802760377) + | > loader_time: 0.014 (0.012358656055048888) + + + --> TIME: 2024-06-25 07:56:41 -- STEP: 329/591 -- GLOBAL_STEP: 18650 + | > loss_disc: 2.6861448287963867 (2.724127447713834) + | > loss_disc_real_0: 0.21139205992221832 (0.21101918137182218) + | > loss_disc_real_1: 0.21232688426971436 (0.22760804477614835) + | > loss_disc_real_2: 0.2322433441877365 (0.23692432920077652) + | > loss_disc_real_3: 0.23991625010967255 (0.23554051769359496) + | > loss_disc_real_4: 0.24888674914836884 (0.23282777298366408) + | > loss_disc_real_5: 0.24221481382846832 (0.23907368469383217) + | > loss_0: 2.6861448287963867 (2.724127447713834) + | > grad_norm_0: tensor(3.9969, device='cuda:0') (tensor(10.0985, device='cuda:0')) + | > loss_gen: 1.9019585847854614 (1.908366180843133) + | > loss_kl: 2.4082531929016113 (2.5243637098005114) + | > loss_feat: 3.6900393962860107 (3.637568278877931) + | > loss_mel: 21.437633514404297 (21.07879450835716) + | > loss_duration: 1.3957877159118652 (1.3560668400355746) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.83367347717285 (30.50515950559483) + | > grad_norm_1: tensor(150.4382, device='cuda:0') (tensor(125.5930, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.9191 (0.8605483187005877) + | > loader_time: 0.0139 (0.01260468880093931) + + + --> TIME: 2024-06-25 07:57:05 -- STEP: 354/591 -- GLOBAL_STEP: 18675 + | > loss_disc: 2.716701030731201 (2.7239340382107233) + | > loss_disc_real_0: 0.16557861864566803 (0.21094823658129588) + | > loss_disc_real_1: 0.2286069244146347 (0.2276775142819868) + | > loss_disc_real_2: 0.23683114349842072 (0.23699143684684892) + | > loss_disc_real_3: 0.2239733338356018 (0.23548796533022898) + | > loss_disc_real_4: 0.2155476212501526 (0.23262203244839685) + | > loss_disc_real_5: 0.23318424820899963 (0.23882309738862312) + | > loss_0: 2.716701030731201 (2.7239340382107233) + | > grad_norm_0: tensor(7.2486, device='cuda:0') (tensor(10.0443, device='cuda:0')) + | > loss_gen: 1.9372327327728271 (1.9079411100533048) + | > loss_kl: 2.5477147102355957 (2.5211375473582813) + | > loss_feat: 3.4888484477996826 (3.6307808315686585) + | > loss_mel: 20.877653121948242 (21.06480319351798) + | > loss_duration: 1.3794909715652466 (1.3572984482608945) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.230939865112305 (30.481961120993404) + | > grad_norm_1: tensor(158.6453, device='cuda:0') (tensor(129.2073, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.9439 (0.866098368908726) + | > loader_time: 0.0167 (0.012881816443750414) + + + --> TIME: 2024-06-25 07:57:29 -- STEP: 379/591 -- GLOBAL_STEP: 18700 + | > loss_disc: 2.7108685970306396 (2.7235605421041122) + | > loss_disc_real_0: 0.16574321687221527 (0.21088030718090037) + | > loss_disc_real_1: 0.20634537935256958 (0.22764890517125344) + | > loss_disc_real_2: 0.22100789844989777 (0.2368423645839213) + | > loss_disc_real_3: 0.2188905030488968 (0.23547255788440755) + | > loss_disc_real_4: 0.26246750354766846 (0.23286441491430226) + | > loss_disc_real_5: 0.2558611333370209 (0.2389280623842355) + | > loss_0: 2.7108685970306396 (2.7235605421041122) + | > grad_norm_0: tensor(8.5893, device='cuda:0') (tensor(9.9730, device='cuda:0')) + | > loss_gen: 1.8759088516235352 (1.9075221304528631) + | > loss_kl: 2.467880964279175 (2.519002464012609) + | > loss_feat: 3.5689501762390137 (3.624199089085521) + | > loss_mel: 20.690988540649414 (21.054226641290082) + | > loss_duration: 1.3766013383865356 (1.3588754482822873) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.980331420898438 (30.463825779414115) + | > grad_norm_1: tensor(64.8238, device='cuda:0') (tensor(130.6705, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.9627 (0.8721979280889508) + | > loader_time: 0.0185 (0.013133451617802038) + + + --> TIME: 2024-06-25 07:57:54 -- STEP: 404/591 -- GLOBAL_STEP: 18725 + | > loss_disc: 2.76578950881958 (2.7228222158875797) + | > loss_disc_real_0: 0.199283167719841 (0.2108264815910618) + | > loss_disc_real_1: 0.22979599237442017 (0.22745578238131978) + | > loss_disc_real_2: 0.2205003947019577 (0.23674434353366935) + | > loss_disc_real_3: 0.2115383893251419 (0.23535296390994942) + | > loss_disc_real_4: 0.24398072063922882 (0.2328492371310102) + | > loss_disc_real_5: 0.23246724903583527 (0.23882991046008498) + | > loss_0: 2.76578950881958 (2.7228222158875797) + | > grad_norm_0: tensor(6.2043, device='cuda:0') (tensor(10.0435, device='cuda:0')) + | > loss_gen: 1.9344298839569092 (1.9074721531112595) + | > loss_kl: 2.446730375289917 (2.517323584249705) + | > loss_feat: 3.5090768337249756 (3.6212093387499893) + | > loss_mel: 20.41425323486328 (21.04381578747587) + | > loss_duration: 1.357088565826416 (1.3600199898280723) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.661577224731445 (30.449840838366217) + | > grad_norm_1: tensor(81.9369, device='cuda:0') (tensor(128.9942, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 1.0082 (0.878811305112178) + | > loader_time: 0.0182 (0.013389577369878787) + + + --> TIME: 2024-06-25 07:58:20 -- STEP: 429/591 -- GLOBAL_STEP: 18750 + | > loss_disc: 2.7208526134490967 (2.7225623147470968) + | > loss_disc_real_0: 0.22568270564079285 (0.21090178087457911) + | > loss_disc_real_1: 0.22360333800315857 (0.22750459826770641) + | > loss_disc_real_2: 0.22470299899578094 (0.23656948490854188) + | > loss_disc_real_3: 0.19649910926818848 (0.23527022199336187) + | > loss_disc_real_4: 0.23477037250995636 (0.23280124519115838) + | > loss_disc_real_5: 0.21903476119041443 (0.2388809226053856) + | > loss_0: 2.7208526134490967 (2.7225623147470968) + | > grad_norm_0: tensor(6.1704, device='cuda:0') (tensor(10.0028, device='cuda:0')) + | > loss_gen: 1.737654209136963 (1.9062016099205106) + | > loss_kl: 2.5199363231658936 (2.5152189286994493) + | > loss_feat: 3.3357315063476562 (3.614030828564873) + | > loss_mel: 20.980148315429688 (21.034859010389617) + | > loss_duration: 1.3721845149993896 (1.3608088729542733) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.945653915405273 (30.431119251918126) + | > grad_norm_1: tensor(158.7390, device='cuda:0') (tensor(127.7626, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 0.9767 (0.8854820794992515) + | > loader_time: 0.0158 (0.013653482312644833) + + + --> TIME: 2024-06-25 07:58:45 -- STEP: 454/591 -- GLOBAL_STEP: 18775 + | > loss_disc: 2.7721712589263916 (2.724430275908651) + | > loss_disc_real_0: 0.2736720144748688 (0.21124583285810664) + | > loss_disc_real_1: 0.24882330000400543 (0.22755945897312416) + | > loss_disc_real_2: 0.2603723704814911 (0.23654380102645983) + | > loss_disc_real_3: 0.2697329819202423 (0.23552026599645615) + | > loss_disc_real_4: 0.23109735548496246 (0.23295696023933687) + | > loss_disc_real_5: 0.27127528190612793 (0.2390086473938127) + | > loss_0: 2.7721712589263916 (2.724430275908651) + | > grad_norm_0: tensor(13.2172, device='cuda:0') (tensor(10.2232, device='cuda:0')) + | > loss_gen: 1.7864344120025635 (1.9047423181029668) + | > loss_kl: 2.424400568008423 (2.5134762467791862) + | > loss_feat: 3.23348331451416 (3.602013083806647) + | > loss_mel: 20.609094619750977 (21.02014242499934) + | > loss_duration: 1.3878014087677002 (1.361886567481289) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.44121551513672 (30.4022606458958) + | > grad_norm_1: tensor(50.4075, device='cuda:0') (tensor(127.5613, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 1.0003 (0.8921554046580447) + | > loader_time: 0.0194 (0.013932868247515304) + + + --> TIME: 2024-06-25 07:59:12 -- STEP: 479/591 -- GLOBAL_STEP: 18800 + | > loss_disc: 2.708373546600342 (2.7244010081121965) + | > loss_disc_real_0: 0.2148570418357849 (0.2111402631801454) + | > loss_disc_real_1: 0.23729123175144196 (0.22743681331392618) + | > loss_disc_real_2: 0.2160760760307312 (0.23640765853241738) + | > loss_disc_real_3: 0.22758610546588898 (0.23548730448790534) + | > loss_disc_real_4: 0.190971240401268 (0.23298316254496326) + | > loss_disc_real_5: 0.20614473521709442 (0.2389876359377624) + | > loss_0: 2.708373546600342 (2.7244010081121965) + | > grad_norm_0: tensor(4.8194, device='cuda:0') (tensor(10.1240, device='cuda:0')) + | > loss_gen: 1.776064157485962 (1.9032777440821702) + | > loss_kl: 2.472404956817627 (2.5126568848006667) + | > loss_feat: 3.4631729125976562 (3.5937992272148054) + | > loss_mel: 21.231718063354492 (21.01915973113821) + | > loss_duration: 1.4273934364318848 (1.3631221757801193) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.370752334594727 (30.392015759780662) + | > grad_norm_1: tensor(88.1063, device='cuda:0') (tensor(127.7439, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 1.0185 (0.8987385878234418) + | > loader_time: 0.0177 (0.014222722959418885) + + + --> TIME: 2024-06-25 07:59:39 -- STEP: 504/591 -- GLOBAL_STEP: 18825 + | > loss_disc: 2.694729804992676 (2.724701143446423) + | > loss_disc_real_0: 0.16237066686153412 (0.2111507784870882) + | > loss_disc_real_1: 0.2210804671049118 (0.22734608418411678) + | > loss_disc_real_2: 0.22738656401634216 (0.2364024460788757) + | > loss_disc_real_3: 0.2525826394557953 (0.235557224483244) + | > loss_disc_real_4: 0.23660293221473694 (0.23312792891547793) + | > loss_disc_real_5: 0.21993014216423035 (0.23907174749506843) + | > loss_0: 2.694729804992676 (2.724701143446423) + | > grad_norm_0: tensor(7.7172, device='cuda:0') (tensor(10.1531, device='cuda:0')) + | > loss_gen: 1.8119568824768066 (1.90239455846567) + | > loss_kl: 2.4652538299560547 (2.5101234699998587) + | > loss_feat: 3.2641892433166504 (3.585813849218308) + | > loss_mel: 21.10967254638672 (21.003633120703306) + | > loss_duration: 1.4048678874969482 (1.3644289130729343) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.055938720703125 (30.366393910513985) + | > grad_norm_1: tensor(95.6302, device='cuda:0') (tensor(126.1870, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 1.0899 (0.9069735502439833) + | > loader_time: 0.0218 (0.014545117105756487) + + + --> TIME: 2024-06-25 08:00:07 -- STEP: 529/591 -- GLOBAL_STEP: 18850 + | > loss_disc: 2.715494394302368 (2.7247170680862745) + | > loss_disc_real_0: 0.2049035131931305 (0.21114703605835292) + | > loss_disc_real_1: 0.1916574090719223 (0.22743355196905948) + | > loss_disc_real_2: 0.22599875926971436 (0.2363681088717538) + | > loss_disc_real_3: 0.23530812561511993 (0.23560287342855746) + | > loss_disc_real_4: 0.2501331567764282 (0.23317237308210145) + | > loss_disc_real_5: 0.25142040848731995 (0.2390134233157441) + | > loss_0: 2.715494394302368 (2.7247170680862745) + | > grad_norm_0: tensor(3.8201, device='cuda:0') (tensor(10.1795, device='cuda:0')) + | > loss_gen: 1.8056409358978271 (1.9024228808559858) + | > loss_kl: 2.500319480895996 (2.509332683902157) + | > loss_feat: 3.39155912399292 (3.579741423432002) + | > loss_mel: 21.032732009887695 (20.995590476773394) + | > loss_duration: 1.396033763885498 (1.366004776413815) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.126285552978516 (30.3530922512927) + | > grad_norm_1: tensor(96.8961, device='cuda:0') (tensor(127.4421, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 1.1135 (0.9157980966658135) + | > loader_time: 0.0194 (0.014859138229168204) + + + --> TIME: 2024-06-25 08:00:36 -- STEP: 554/591 -- GLOBAL_STEP: 18875 + | > loss_disc: 2.73260235786438 (2.7243700449217094) + | > loss_disc_real_0: 0.22479139268398285 (0.21096146858502388) + | > loss_disc_real_1: 0.17552417516708374 (0.22748151843836162) + | > loss_disc_real_2: 0.23575851321220398 (0.23626436309263593) + | > loss_disc_real_3: 0.24570271372795105 (0.23552447974359086) + | > loss_disc_real_4: 0.26287585496902466 (0.23321684202455012) + | > loss_disc_real_5: 0.23454105854034424 (0.23905063791718292) + | > loss_0: 2.73260235786438 (2.7243700449217094) + | > grad_norm_0: tensor(4.1383, device='cuda:0') (tensor(10.0986, device='cuda:0')) + | > loss_gen: 1.8687043190002441 (1.901695791134335) + | > loss_kl: 2.5676605701446533 (2.506419868675811) + | > loss_feat: 3.2211389541625977 (3.572266319168173) + | > loss_mel: 20.301639556884766 (20.978421638158242) + | > loss_duration: 1.403895378112793 (1.36749707920887) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.363037109375 (30.32630070021867) + | > grad_norm_1: tensor(66.3479, device='cuda:0') (tensor(126.7437, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 1.1157 (0.9267862023859679) + | > loader_time: 0.0227 (0.015189972164828855) + + + --> TIME: 2024-06-25 08:01:09 -- STEP: 579/591 -- GLOBAL_STEP: 18900 + | > loss_disc: 2.762634754180908 (2.7250323427361725) + | > loss_disc_real_0: 0.10385973006486893 (0.21092101182653505) + | > loss_disc_real_1: 0.1996528059244156 (0.22759762051191973) + | > loss_disc_real_2: 0.21267791092395782 (0.23622030606541608) + | > loss_disc_real_3: 0.1932285875082016 (0.23550827796080365) + | > loss_disc_real_4: 0.24690227210521698 (0.23320838796145352) + | > loss_disc_real_5: 0.24646052718162537 (0.23908857566293246) + | > loss_0: 2.762634754180908 (2.7250323427361725) + | > grad_norm_0: tensor(26.2965, device='cuda:0') (tensor(10.1327, device='cuda:0')) + | > loss_gen: 2.1079745292663574 (1.9010821496472663) + | > loss_kl: 2.420961618423462 (2.503600134133058) + | > loss_feat: 3.4696483612060547 (3.5642089440209053) + | > loss_mel: 20.717811584472656 (20.958492097871076) + | > loss_duration: 1.408355474472046 (1.3690628612185596) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.124752044677734 (30.29644619903828) + | > grad_norm_1: tensor(261.7233, device='cuda:0') (tensor(127.7637, device='cuda:0')) + | > current_lr_0: 0.00019922645137067577 + | > current_lr_1: 0.00019922645137067577 + | > step_time: 1.329 (0.9416404577528669) + | > loader_time: 0.0265 (0.01557492715707097) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003655332200070645 (-8.22239733756858e-05) + | > avg_loss_disc: 2.6914270583619464 (-0.04029977575261512) + | > avg_loss_disc_real_0: 0.2018573414138023 (-0.0841014055495567) + | > avg_loss_disc_real_1: 0.22544732689857483 (-0.02723197385351711) + | > avg_loss_disc_real_2: 0.23656945089076428 (+0.0027905391885879316) + | > avg_loss_disc_real_3: 0.2854330701396819 (+0.04700138911287821) + | > avg_loss_disc_real_4: 0.2551730693654812 (-0.0014705889402551464) + | > avg_loss_disc_real_5: 0.24460723425479644 (-0.0014216915724125367) + | > avg_loss_0: 2.6914270583619464 (-0.04029977575261512) + | > avg_loss_gen: 1.9658558698410684 (-0.0770197523401146) + | > avg_loss_kl: 2.5171737569443717 (-0.0809119102802689) + | > avg_loss_feat: 3.4639123703571078 (+0.017018876177199438) + | > avg_loss_mel: 20.772922637614798 (+0.2603453778206095) + | > avg_loss_duration: 1.3891880512237542 (+0.014633627648048542) + | > avg_loss_1: 30.10905261749917 (+0.13406607445250174) + + + > EPOCH: 33/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 08:01:33)  + + --> TIME: 2024-06-25 08:01:44 -- STEP: 13/591 -- GLOBAL_STEP: 18925 + | > loss_disc: 2.70963716506958 (2.719829742725079) + | > loss_disc_real_0: 0.2367163896560669 (0.21127641315643603) + | > loss_disc_real_1: 0.23103119432926178 (0.2257226464840082) + | > loss_disc_real_2: 0.222870871424675 (0.22721055723153627) + | > loss_disc_real_3: 0.23531469702720642 (0.2363334412758167) + | > loss_disc_real_4: 0.21711553633213043 (0.23538134877498335) + | > loss_disc_real_5: 0.20873941481113434 (0.2440356852916571) + | > loss_0: 2.70963716506958 (2.719829742725079) + | > grad_norm_0: tensor(8.9825, device='cuda:0') (tensor(9.8712, device='cuda:0')) + | > loss_gen: 1.888169765472412 (1.9274662457979643) + | > loss_kl: 2.6019954681396484 (2.678784260383019) + | > loss_feat: 3.7668702602386475 (3.8118680257063646) + | > loss_mel: 21.043121337890625 (21.077857237595772) + | > loss_duration: 1.334350347518921 (1.3334203683412993) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.63450813293457 (30.829396174504208) + | > grad_norm_1: tensor(43.0606, device='cuda:0') (tensor(133.3747, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.7892 (0.7694008166973407) + | > loader_time: 0.0092 (0.008666075192964993) + + + --> TIME: 2024-06-25 08:02:05 -- STEP: 38/591 -- GLOBAL_STEP: 18950 + | > loss_disc: 2.7926688194274902 (2.7431394112737557) + | > loss_disc_real_0: 0.28823405504226685 (0.21475555041902944) + | > loss_disc_real_1: 0.21715205907821655 (0.22978377146156212) + | > loss_disc_real_2: 0.2097969949245453 (0.2386222685638227) + | > loss_disc_real_3: 0.19308027625083923 (0.23598223806996094) + | > loss_disc_real_4: 0.24075281620025635 (0.23497262008880315) + | > loss_disc_real_5: 0.26397791504859924 (0.23958093633777217) + | > loss_0: 2.7926688194274902 (2.7431394112737557) + | > grad_norm_0: tensor(14.4960, device='cuda:0') (tensor(11.0504, device='cuda:0')) + | > loss_gen: 1.823622703552246 (1.9156855313401473) + | > loss_kl: 2.694701910018921 (2.6499633349870377) + | > loss_feat: 3.5542798042297363 (3.7108062380238582) + | > loss_mel: 21.32583236694336 (21.082263695566272) + | > loss_duration: 1.334801197052002 (1.3349780377588776) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.733238220214844 (30.693696875321237) + | > grad_norm_1: tensor(87.7439, device='cuda:0') (tensor(110.8448, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.7961 (0.7842679964868646) + | > loader_time: 0.0096 (0.009422929663407173) + + + --> TIME: 2024-06-25 08:02:25 -- STEP: 63/591 -- GLOBAL_STEP: 18975 + | > loss_disc: 2.7252650260925293 (2.7327791244264636) + | > loss_disc_real_0: 0.18193398416042328 (0.2117573323230895) + | > loss_disc_real_1: 0.19491338729858398 (0.2278828528665361) + | > loss_disc_real_2: 0.24264273047447205 (0.23771136573382787) + | > loss_disc_real_3: 0.21343830227851868 (0.23563148601660652) + | > loss_disc_real_4: 0.20694896578788757 (0.23442734446790484) + | > loss_disc_real_5: 0.2350214570760727 (0.24039557363305772) + | > loss_0: 2.7252650260925293 (2.7327791244264636) + | > grad_norm_0: tensor(6.2318, device='cuda:0') (tensor(10.0902, device='cuda:0')) + | > loss_gen: 1.9206273555755615 (1.912115770672995) + | > loss_kl: 2.4460246562957764 (2.608913028050982) + | > loss_feat: 3.5467700958251953 (3.6947055771237327) + | > loss_mel: 21.00528907775879 (21.07200340997605) + | > loss_duration: 1.3719619512557983 (1.344322346505665) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.290674209594727 (30.632060126652792) + | > grad_norm_1: tensor(54.4213, device='cuda:0') (tensor(106.7099, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.8255 (0.795965762365432) + | > loader_time: 0.0117 (0.009939424575321255) + + + --> TIME: 2024-06-25 08:02:46 -- STEP: 88/591 -- GLOBAL_STEP: 19000 + | > loss_disc: 2.7300260066986084 (2.7286816347729075) + | > loss_disc_real_0: 0.11148104071617126 (0.21096844403919848) + | > loss_disc_real_1: 0.18780052661895752 (0.2269912213087082) + | > loss_disc_real_2: 0.18065840005874634 (0.23750585097480903) + | > loss_disc_real_3: 0.21487942337989807 (0.23507421192797748) + | > loss_disc_real_4: 0.20383235812187195 (0.2337908141992309) + | > loss_disc_real_5: 0.2107919305562973 (0.2391997528347102) + | > loss_0: 2.7300260066986084 (2.7286816347729075) + | > grad_norm_0: tensor(19.2757, device='cuda:0') (tensor(10.7679, device='cuda:0')) + | > loss_gen: 2.2653937339782715 (1.915938365188512) + | > loss_kl: 2.617914915084839 (2.592114052989265) + | > loss_feat: 4.139246940612793 (3.7023090449246494) + | > loss_mel: 21.773109436035156 (21.10445495085282) + | > loss_duration: 1.409880518913269 (1.3514511476863516) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 32.20554733276367 (30.666267568414863) + | > grad_norm_1: tensor(159.0629, device='cuda:0') (tensor(122.0889, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.8464 (0.8037155649878762) + | > loader_time: 0.0116 (0.010225967927412554) + + + --> TIME: 2024-06-25 08:03:07 -- STEP: 113/591 -- GLOBAL_STEP: 19025 + | > loss_disc: 2.710836887359619 (2.7268631922460234) + | > loss_disc_real_0: 0.16529622673988342 (0.21117963549573865) + | > loss_disc_real_1: 0.17687611281871796 (0.227387099271327) + | > loss_disc_real_2: 0.2441871017217636 (0.23722641222772345) + | > loss_disc_real_3: 0.23086820542812347 (0.23515389688247074) + | > loss_disc_real_4: 0.275190532207489 (0.23427789348416625) + | > loss_disc_real_5: 0.25631183385849 (0.2393715596568268) + | > loss_0: 2.710836887359619 (2.7268631922460234) + | > grad_norm_0: tensor(7.6136, device='cuda:0') (tensor(10.4419, device='cuda:0')) + | > loss_gen: 2.001913547515869 (1.9110280338641816) + | > loss_kl: 2.515730619430542 (2.5704045295715323) + | > loss_feat: 3.6453495025634766 (3.6827817342977607) + | > loss_mel: 20.56133460998535 (21.072924740546565) + | > loss_duration: 1.3460297584533691 (1.3549640442417792) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.070358276367188 (30.59210299601597) + | > grad_norm_1: tensor(202.6426, device='cuda:0') (tensor(119.5513, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.839 (0.8104581305411009) + | > loader_time: 0.0108 (0.010386572474926971) + + + --> TIME: 2024-06-25 08:03:29 -- STEP: 138/591 -- GLOBAL_STEP: 19050 + | > loss_disc: 2.75602650642395 (2.7255860273388848) + | > loss_disc_real_0: 0.29592037200927734 (0.2121493088054484) + | > loss_disc_real_1: 0.2137698084115982 (0.2273049971115762) + | > loss_disc_real_2: 0.21718111634254456 (0.23630827956873438) + | > loss_disc_real_3: 0.2645941376686096 (0.23543861378794131) + | > loss_disc_real_4: 0.22002829611301422 (0.23349596667980801) + | > loss_disc_real_5: 0.23426926136016846 (0.23915920330994372) + | > loss_0: 2.75602650642395 (2.7255860273388848) + | > grad_norm_0: tensor(12.2644, device='cuda:0') (tensor(10.6293, device='cuda:0')) + | > loss_gen: 1.8979805707931519 (1.9100589372109675) + | > loss_kl: 2.5708653926849365 (2.564487681872602) + | > loss_feat: 3.6442089080810547 (3.677535146906756) + | > loss_mel: 21.450780868530273 (21.057441006536063) + | > loss_duration: 1.3760912418365479 (1.3557294011116032) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.939926147460938 (30.565252082935277) + | > grad_norm_1: tensor(105.0781, device='cuda:0') (tensor(120.3807, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.8532 (0.8168930471807286) + | > loader_time: 0.0123 (0.010549688684767569) + + + --> TIME: 2024-06-25 08:03:51 -- STEP: 163/591 -- GLOBAL_STEP: 19075 + | > loss_disc: 2.6847798824310303 (2.726463272527683) + | > loss_disc_real_0: 0.19904814660549164 (0.21198757271642335) + | > loss_disc_real_1: 0.24725249409675598 (0.22777505337826312) + | > loss_disc_real_2: 0.24571408331394196 (0.23628875521794418) + | > loss_disc_real_3: 0.22205539047718048 (0.23535677927769036) + | > loss_disc_real_4: 0.22559404373168945 (0.23332980019183247) + | > loss_disc_real_5: 0.2241096943616867 (0.23906102310294755) + | > loss_0: 2.6847798824310303 (2.726463272527683) + | > grad_norm_0: tensor(3.2141, device='cuda:0') (tensor(10.4242, device='cuda:0')) + | > loss_gen: 1.886411190032959 (1.9084057061950122) + | > loss_kl: 2.608193874359131 (2.5574421868002477) + | > loss_feat: 3.6199731826782227 (3.6711131019826317) + | > loss_mel: 21.132850646972656 (21.030969526138772) + | > loss_duration: 1.3518919944763184 (1.3539078250253131) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.599321365356445 (30.521838299335876) + | > grad_norm_1: tensor(53.8060, device='cuda:0') (tensor(116.8299, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.8542 (0.8227449019262396) + | > loader_time: 0.0125 (0.010735144644427153) + + + --> TIME: 2024-06-25 08:04:13 -- STEP: 188/591 -- GLOBAL_STEP: 19100 + | > loss_disc: 2.7444796562194824 (2.725354988524253) + | > loss_disc_real_0: 0.35646387934684753 (0.21170987866501859) + | > loss_disc_real_1: 0.19495460391044617 (0.2277720293149035) + | > loss_disc_real_2: 0.23624888062477112 (0.23626018347258262) + | > loss_disc_real_3: 0.24962659180164337 (0.23538581694060184) + | > loss_disc_real_4: 0.20177902281284332 (0.2333685332473288) + | > loss_disc_real_5: 0.22070744633674622 (0.23918964697959574) + | > loss_0: 2.7444796562194824 (2.725354988524253) + | > grad_norm_0: tensor(26.9384, device='cuda:0') (tensor(10.2427, device='cuda:0')) + | > loss_gen: 1.6629704236984253 (1.9076443384302424) + | > loss_kl: 2.4484786987304688 (2.5542674850910263) + | > loss_feat: 3.765141010284424 (3.6679091174551783) + | > loss_mel: 21.714845657348633 (21.00936362083923) + | > loss_duration: 1.357175588607788 (1.3522876336219467) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.948610305786133 (30.49147213266251) + | > grad_norm_1: tensor(66.9401, device='cuda:0') (tensor(117.3935, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.8724 (0.8283053433641474) + | > loader_time: 0.0134 (0.010975746398276476) + + + --> TIME: 2024-06-25 08:04:35 -- STEP: 213/591 -- GLOBAL_STEP: 19125 + | > loss_disc: 2.7315495014190674 (2.7269090106229816) + | > loss_disc_real_0: 0.21727591753005981 (0.21226033019208013) + | > loss_disc_real_1: 0.26203131675720215 (0.22793193539263498) + | > loss_disc_real_2: 0.25783851742744446 (0.23641738085679606) + | > loss_disc_real_3: 0.2082851678133011 (0.2351859427393882) + | > loss_disc_real_4: 0.247421532869339 (0.23339242050905182) + | > loss_disc_real_5: 0.2421862930059433 (0.23896420484697314) + | > loss_0: 2.7315495014190674 (2.7269090106229816) + | > grad_norm_0: tensor(6.1486, device='cuda:0') (tensor(10.6112, device='cuda:0')) + | > loss_gen: 1.872666835784912 (1.9078809224383932) + | > loss_kl: 2.5615439414978027 (2.5506380186394337) + | > loss_feat: 3.3090572357177734 (3.660195202894614) + | > loss_mel: 20.85260772705078 (21.005727132161468) + | > loss_duration: 1.3524625301361084 (1.3514248082335572) + | > amp_scaler: 512.0 (528.8262910798121) + | > loss_1: 29.948339462280273 (30.475866040153683) + | > grad_norm_1: tensor(229.7318, device='cuda:0') (tensor(120.6883, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.8722 (0.8331514065254462) + | > loader_time: 0.012 (0.011226782776380373) + + + --> TIME: 2024-06-25 08:04:57 -- STEP: 238/591 -- GLOBAL_STEP: 19150 + | > loss_disc: 2.6468729972839355 (2.723520973149467) + | > loss_disc_real_0: 0.15173453092575073 (0.21155366010400428) + | > loss_disc_real_1: 0.2140604555606842 (0.22761202484619716) + | > loss_disc_real_2: 0.22109095752239227 (0.23586301849920208) + | > loss_disc_real_3: 0.25448644161224365 (0.2350216415248999) + | > loss_disc_real_4: 0.22883416712284088 (0.23312574506056408) + | > loss_disc_real_5: 0.22115570306777954 (0.23884006220252574) + | > loss_0: 2.6468729972839355 (2.723520973149467) + | > grad_norm_0: tensor(10.9918, device='cuda:0') (tensor(10.4901, device='cuda:0')) + | > loss_gen: 1.8544692993164062 (1.9086805231430952) + | > loss_kl: 2.649472951889038 (2.5501944658135156) + | > loss_feat: 3.952911615371704 (3.662970529884851) + | > loss_mel: 21.560914993286133 (21.023406894267115) + | > loss_duration: 1.3147791624069214 (1.3513679068629485) + | > amp_scaler: 512.0 (527.0588235294117) + | > loss_1: 31.332548141479492 (30.496620274391496) + | > grad_norm_1: tensor(72.1063, device='cuda:0') (tensor(118.8491, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.8739 (0.8381338490157568) + | > loader_time: 0.0136 (0.011534107833349409) + + + --> TIME: 2024-06-25 08:05:20 -- STEP: 263/591 -- GLOBAL_STEP: 19175 + | > loss_disc: 2.7263078689575195 (2.722547941788067) + | > loss_disc_real_0: 0.18179123103618622 (0.21128776853642084) + | > loss_disc_real_1: 0.23274876177310944 (0.22770148141755803) + | > loss_disc_real_2: 0.24365492165088654 (0.2357785830366294) + | > loss_disc_real_3: 0.229616180062294 (0.2349483918101162) + | > loss_disc_real_4: 0.19784995913505554 (0.23315023667685433) + | > loss_disc_real_5: 0.22525346279144287 (0.2389710665089096) + | > loss_0: 2.7263078689575195 (2.722547941788067) + | > grad_norm_0: tensor(7.3455, device='cuda:0') (tensor(10.3273, device='cuda:0')) + | > loss_gen: 1.85471773147583 (1.909982952781503) + | > loss_kl: 2.5232319831848145 (2.5467535950838376) + | > loss_feat: 3.6132445335388184 (3.6674432192465196) + | > loss_mel: 20.849655151367188 (21.036952268941327) + | > loss_duration: 1.3444936275482178 (1.3514214237379942) + | > amp_scaler: 512.0 (525.6273764258555) + | > loss_1: 30.18534278869629 (30.512553370950794) + | > grad_norm_1: tensor(101.0285, device='cuda:0') (tensor(118.2640, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.9049 (0.8435197429511937) + | > loader_time: 0.0156 (0.01188200870847521) + + + --> TIME: 2024-06-25 08:05:43 -- STEP: 288/591 -- GLOBAL_STEP: 19200 + | > loss_disc: 2.7163825035095215 (2.7236286368634954) + | > loss_disc_real_0: 0.17282432317733765 (0.21131049890795517) + | > loss_disc_real_1: 0.20628127455711365 (0.22745740962111294) + | > loss_disc_real_2: 0.22844892740249634 (0.2355144828454488) + | > loss_disc_real_3: 0.233427032828331 (0.23590598974583876) + | > loss_disc_real_4: 0.231539785861969 (0.23305708252721363) + | > loss_disc_real_5: 0.20106184482574463 (0.2387644053540296) + | > loss_0: 2.7163825035095215 (2.7236286368634954) + | > grad_norm_0: tensor(7.1798, device='cuda:0') (tensor(10.3288, device='cuda:0')) + | > loss_gen: 1.980759620666504 (1.9088506686190765) + | > loss_kl: 2.533034563064575 (2.5444034031695772) + | > loss_feat: 3.6051828861236572 (3.6578831035229893) + | > loss_mel: 21.26784896850586 (21.035738017823967) + | > loss_duration: 1.3945426940917969 (1.3524657152593142) + | > amp_scaler: 512.0 (524.4444444444443) + | > loss_1: 30.781368255615234 (30.499340838856167) + | > grad_norm_1: tensor(226.9106, device='cuda:0') (tensor(124.1377, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.8957 (0.8486685653527578) + | > loader_time: 0.0141 (0.012158006429672246) + + + --> TIME: 2024-06-25 08:06:07 -- STEP: 313/591 -- GLOBAL_STEP: 19225 + | > loss_disc: 2.706547975540161 (2.7238472719162026) + | > loss_disc_real_0: 0.1601928323507309 (0.21116390812416055) + | > loss_disc_real_1: 0.24003471434116364 (0.22769184448658092) + | > loss_disc_real_2: 0.24671445786952972 (0.23549848337904714) + | > loss_disc_real_3: 0.209537073969841 (0.23599022750656445) + | > loss_disc_real_4: 0.2046949863433838 (0.23319441941789926) + | > loss_disc_real_5: 0.2158413976430893 (0.23895239201597512) + | > loss_0: 2.706547975540161 (2.7238472719162026) + | > grad_norm_0: tensor(9.9153, device='cuda:0') (tensor(10.3121, device='cuda:0')) + | > loss_gen: 1.9241039752960205 (1.9080645988543574) + | > loss_kl: 2.5172722339630127 (2.5413191859333644) + | > loss_feat: 3.441171646118164 (3.651885146911914) + | > loss_mel: 20.752962112426758 (21.024874952273624) + | > loss_duration: 1.3485324382781982 (1.3528034260478656) + | > amp_scaler: 512.0 (523.4504792332268) + | > loss_1: 29.98404312133789 (30.47894723422992) + | > grad_norm_1: tensor(272.1963, device='cuda:0') (tensor(123.8748, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.9137 (0.8542255837315569) + | > loader_time: 0.0157 (0.01241609235160268) + + + --> TIME: 2024-06-25 08:06:31 -- STEP: 338/591 -- GLOBAL_STEP: 19250 + | > loss_disc: 2.7539923191070557 (2.7235113220101956) + | > loss_disc_real_0: 0.3028147220611572 (0.2113381273295048) + | > loss_disc_real_1: 0.22754541039466858 (0.22788588901066925) + | > loss_disc_real_2: 0.23019060492515564 (0.23543391173936912) + | > loss_disc_real_3: 0.24999761581420898 (0.23604553523500996) + | > loss_disc_real_4: 0.23068256676197052 (0.23306487844540522) + | > loss_disc_real_5: 0.2236335128545761 (0.2386026407575466) + | > loss_0: 2.7539923191070557 (2.7235113220101956) + | > grad_norm_0: tensor(15.5453, device='cuda:0') (tensor(10.5019, device='cuda:0')) + | > loss_gen: 1.8885259628295898 (1.9085355647922269) + | > loss_kl: 2.522757053375244 (2.536299228668211) + | > loss_feat: 3.7412161827087402 (3.6469761902058617) + | > loss_mel: 21.82771873474121 (21.031874724393756) + | > loss_duration: 1.405977487564087 (1.3540063034853296) + | > amp_scaler: 512.0 (522.6035502958581) + | > loss_1: 31.386194229125977 (30.477691943828876) + | > grad_norm_1: tensor(269.9579, device='cuda:0') (tensor(131.1563, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.9175 (0.8596895418223545) + | > loader_time: 0.0144 (0.012671154631665483) + + + --> TIME: 2024-06-25 08:06:55 -- STEP: 363/591 -- GLOBAL_STEP: 19275 + | > loss_disc: 3.0046472549438477 (2.724884339928953) + | > loss_disc_real_0: 0.18791638314723969 (0.210918936437155) + | > loss_disc_real_1: 0.43191877007484436 (0.22926569210760525) + | > loss_disc_real_2: 0.37401050329208374 (0.23627396432821415) + | > loss_disc_real_3: 0.2348131239414215 (0.23631188158489785) + | > loss_disc_real_4: 0.2530628740787506 (0.23304352833383996) + | > loss_disc_real_5: 0.291273832321167 (0.23865982578148856) + | > loss_0: 3.0046472549438477 (2.724884339928953) + | > grad_norm_0: tensor(15.0835, device='cuda:0') (tensor(10.6023, device='cuda:0')) + | > loss_gen: 1.967376708984375 (1.918084466424527) + | > loss_kl: 2.592982530593872 (2.533356087266905) + | > loss_feat: 3.906935691833496 (3.660211474442285) + | > loss_mel: 22.6257266998291 (21.057645634842963) + | > loss_duration: 1.377199649810791 (1.3558287262588176) + | > amp_scaler: 256.0 (513.4104683195594) + | > loss_1: 32.47022247314453 (30.525126344244672) + | > grad_norm_1: tensor(288.6476, device='cuda:0') (tensor(142.6743, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.9369 (0.8658836919085711) + | > loader_time: 0.0152 (0.01291438502056869) + + + --> TIME: 2024-06-25 08:07:19 -- STEP: 388/591 -- GLOBAL_STEP: 19300 + | > loss_disc: 2.7550835609436035 (2.732560799293908) + | > loss_disc_real_0: 0.22787217795848846 (0.21121043268322337) + | > loss_disc_real_1: 0.2587529718875885 (0.23381663837767758) + | > loss_disc_real_2: 0.21738110482692719 (0.23709977733105728) + | > loss_disc_real_3: 0.2114311307668686 (0.23636685834103024) + | > loss_disc_real_4: 0.2345752865076065 (0.23378333948629418) + | > loss_disc_real_5: 0.25684377551078796 (0.2387189477221253) + | > loss_0: 2.7550835609436035 (2.732560799293908) + | > grad_norm_0: tensor(4.3721, device='cuda:0') (tensor(10.8135, device='cuda:0')) + | > loss_gen: 1.7832423448562622 (1.927798718828516) + | > loss_kl: 2.4839062690734863 (2.5299686058280386) + | > loss_feat: 3.283463954925537 (3.6674163353811835) + | > loss_mel: 20.931812286376953 (21.08237100384899) + | > loss_duration: 1.3500349521636963 (1.3570062644088403) + | > amp_scaler: 256.0 (496.8247422680413) + | > loss_1: 29.832460403442383 (30.56456090002945) + | > grad_norm_1: tensor(209.2791, device='cuda:0') (tensor(151.7751, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.9668 (0.8719598091754717) + | > loader_time: 0.0156 (0.01316451964919101) + + + --> TIME: 2024-06-25 08:07:44 -- STEP: 413/591 -- GLOBAL_STEP: 19325 + | > loss_disc: 2.7130653858184814 (2.7342137164700087) + | > loss_disc_real_0: 0.2121259570121765 (0.21141428729523765) + | > loss_disc_real_1: 0.20496341586112976 (0.23387369627164586) + | > loss_disc_real_2: 0.2103937268257141 (0.23690526176595803) + | > loss_disc_real_3: 0.22082483768463135 (0.2360496001800671) + | > loss_disc_real_4: 0.23372361063957214 (0.23392058981099947) + | > loss_disc_real_5: 0.214207261800766 (0.2386599367813683) + | > loss_0: 2.7130653858184814 (2.7342137164700087) + | > grad_norm_0: tensor(3.2862, device='cuda:0') (tensor(10.8826, device='cuda:0')) + | > loss_gen: 1.914424180984497 (1.9221053368820116) + | > loss_kl: 2.4707062244415283 (2.525570534043392) + | > loss_feat: 3.5810744762420654 (3.6424215596178255) + | > loss_mel: 20.92778778076172 (21.061083934497603) + | > loss_duration: 1.357825517654419 (1.3579162365010529) + | > amp_scaler: 256.0 (482.2469733656175) + | > loss_1: 30.251819610595703 (30.509097600387314) + | > grad_norm_1: tensor(75.9247, device='cuda:0') (tensor(147.3419, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 0.9726 (0.8781006740311444) + | > loader_time: 0.0195 (0.013458911212246882) + + + --> TIME: 2024-06-25 08:08:10 -- STEP: 438/591 -- GLOBAL_STEP: 19350 + | > loss_disc: 2.672628402709961 (2.7330329102468243) + | > loss_disc_real_0: 0.20449195802211761 (0.21140011015548016) + | > loss_disc_real_1: 0.2074432075023651 (0.23347409629182186) + | > loss_disc_real_2: 0.23625443875789642 (0.23671535297071553) + | > loss_disc_real_3: 0.2076529860496521 (0.23591840801309777) + | > loss_disc_real_4: 0.22229549288749695 (0.23372710996294674) + | > loss_disc_real_5: 0.236667662858963 (0.23878230311010526) + | > loss_0: 2.672628402709961 (2.7330329102468243) + | > grad_norm_0: tensor(3.6343, device='cuda:0') (tensor(10.7950, device='cuda:0')) + | > loss_gen: 1.8462753295898438 (1.9190851792897263) + | > loss_kl: 2.5284194946289062 (2.5216538372649437) + | > loss_feat: 3.4392359256744385 (3.62816652859727) + | > loss_mel: 21.448440551757812 (21.043801808466107) + | > loss_duration: 1.399107575416565 (1.3589992836185796) + | > amp_scaler: 256.0 (469.3333333333334) + | > loss_1: 30.66147804260254 (30.471706634242786) + | > grad_norm_1: tensor(34.4004, device='cuda:0') (tensor(143.4691, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 1.0278 (0.8847673146147707) + | > loader_time: 0.0186 (0.01372183893369214) + + + --> TIME: 2024-06-25 08:08:36 -- STEP: 463/591 -- GLOBAL_STEP: 19375 + | > loss_disc: 2.7135839462280273 (2.7317825980629284) + | > loss_disc_real_0: 0.11150383949279785 (0.21117191335294988) + | > loss_disc_real_1: 0.2298479676246643 (0.23292929344442445) + | > loss_disc_real_2: 0.2617509663105011 (0.23656373948953063) + | > loss_disc_real_3: 0.26801472902297974 (0.23592449396398105) + | > loss_disc_real_4: 0.18716269731521606 (0.23356107508132062) + | > loss_disc_real_5: 0.21032336354255676 (0.23870281163868584) + | > loss_0: 2.7135839462280273 (2.7317825980629284) + | > grad_norm_0: tensor(19.6924, device='cuda:0') (tensor(10.7713, device='cuda:0')) + | > loss_gen: 2.039381265640259 (1.9181855044148652) + | > loss_kl: 2.5241708755493164 (2.520284435404041) + | > loss_feat: 3.7701356410980225 (3.619593340171336) + | > loss_mel: 21.22185516357422 (21.029318968111703) + | > loss_duration: 1.3708863258361816 (1.360027957940978) + | > amp_scaler: 256.0 (457.8142548596113) + | > loss_1: 30.926429748535156 (30.447410208617637) + | > grad_norm_1: tensor(40.0492, device='cuda:0') (tensor(140.8890, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 1.0247 (0.8914260436855176) + | > loader_time: 0.0188 (0.013990432603312885) + + + --> TIME: 2024-06-25 08:09:02 -- STEP: 488/591 -- GLOBAL_STEP: 19400 + | > loss_disc: 2.716562509536743 (2.7312167736350483) + | > loss_disc_real_0: 0.16184934973716736 (0.2112229519111456) + | > loss_disc_real_1: 0.22030220925807953 (0.23252338523686425) + | > loss_disc_real_2: 0.21279743313789368 (0.2364560766908966) + | > loss_disc_real_3: 0.2501063048839569 (0.23593044467270374) + | > loss_disc_real_4: 0.2122056633234024 (0.2334908077523845) + | > loss_disc_real_5: 0.21999265253543854 (0.23880601338431484) + | > loss_0: 2.716562509536743 (2.7312167736350483) + | > grad_norm_0: tensor(10.0964, device='cuda:0') (tensor(10.7491, device='cuda:0')) + | > loss_gen: 1.995705008506775 (1.9170548222104058) + | > loss_kl: 2.454939126968384 (2.516689559963882) + | > loss_feat: 3.4744482040405273 (3.6106038523502035) + | > loss_mel: 20.87488555908203 (21.01866099091827) + | > loss_duration: 1.4175081253051758 (1.3615390370126639) + | > amp_scaler: 256.0 (447.47540983606564) + | > loss_1: 30.217483520507812 (30.424548266363924) + | > grad_norm_1: tensor(191.4019, device='cuda:0') (tensor(138.6920, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 1.0367 (0.898366349642394) + | > loader_time: 0.0187 (0.014280675376047861) + + + --> TIME: 2024-06-25 08:09:29 -- STEP: 513/591 -- GLOBAL_STEP: 19425 + | > loss_disc: 2.7365145683288574 (2.7319698752018415) + | > loss_disc_real_0: 0.15508228540420532 (0.21127587874904716) + | > loss_disc_real_1: 0.22940212488174438 (0.23246049574534566) + | > loss_disc_real_2: 0.26254695653915405 (0.23657866353877108) + | > loss_disc_real_3: 0.22305113077163696 (0.23590866012996167) + | > loss_disc_real_4: 0.24113421142101288 (0.23363925002471744) + | > loss_disc_real_5: 0.2388685792684555 (0.23892808500786275) + | > loss_0: 2.7365145683288574 (2.7319698752018415) + | > grad_norm_0: tensor(11.7201, device='cuda:0') (tensor(10.7291, device='cuda:0')) + | > loss_gen: 2.0484619140625 (1.9150471266714917) + | > loss_kl: 2.46380352973938 (2.514243600428917) + | > loss_feat: 3.391491413116455 (3.6007315847608776) + | > loss_mel: 20.72811508178711 (21.005529578433865) + | > loss_duration: 1.3986936807632446 (1.363186787908306) + | > amp_scaler: 256.0 (438.1442495126706) + | > loss_1: 30.03056526184082 (30.3987386677465) + | > grad_norm_1: tensor(183.3704, device='cuda:0') (tensor(138.4676, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 1.0834 (0.9060421031818056) + | > loader_time: 0.019 (0.01460376073974848) + + + --> TIME: 2024-06-25 08:09:57 -- STEP: 538/591 -- GLOBAL_STEP: 19450 + | > loss_disc: 2.7469162940979004 (2.731442282634151) + | > loss_disc_real_0: 0.17106929421424866 (0.21126354827812177) + | > loss_disc_real_1: 0.1933596432209015 (0.23219410306514418) + | > loss_disc_real_2: 0.24301238358020782 (0.23666072983059294) + | > loss_disc_real_3: 0.22936983406543732 (0.23590667670433407) + | > loss_disc_real_4: 0.23660123348236084 (0.2335915303474022) + | > loss_disc_real_5: 0.2395879328250885 (0.23892573497109254) + | > loss_0: 2.7469162940979004 (2.731442282634151) + | > grad_norm_0: tensor(7.6985, device='cuda:0') (tensor(10.6461, device='cuda:0')) + | > loss_gen: 1.8104934692382812 (1.913051389407048) + | > loss_kl: 2.3809473514556885 (2.5110985310990563) + | > loss_feat: 3.1645538806915283 (3.5911001636193145) + | > loss_mel: 20.313447952270508 (20.986701887336363) + | > loss_duration: 1.3991758823394775 (1.3647047132807602) + | > amp_scaler: 256.0 (429.68029739776944) + | > loss_1: 29.068618774414062 (30.36665668274834) + | > grad_norm_1: tensor(52.7432, device='cuda:0') (tensor(138.7257, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 1.0916 (0.9144367187882886) + | > loader_time: 0.0231 (0.01494983669550446) + + + --> TIME: 2024-06-25 08:10:26 -- STEP: 563/591 -- GLOBAL_STEP: 19475 + | > loss_disc: 2.7066917419433594 (2.7315169731634734) + | > loss_disc_real_0: 0.2057884782552719 (0.211318324834479) + | > loss_disc_real_1: 0.22126434743404388 (0.23191936707147273) + | > loss_disc_real_2: 0.23189616203308105 (0.236716469323021) + | > loss_disc_real_3: 0.2598053514957428 (0.2359324959677447) + | > loss_disc_real_4: 0.2434563934803009 (0.23355915978365638) + | > loss_disc_real_5: 0.24173812568187714 (0.2389852051254276) + | > loss_0: 2.7066917419433594 (2.7315169731634734) + | > grad_norm_0: tensor(4.6775, device='cuda:0') (tensor(10.7085, device='cuda:0')) + | > loss_gen: 1.955579400062561 (1.9124451746525604) + | > loss_kl: 2.501983404159546 (2.5084560535729024) + | > loss_feat: 3.2462241649627686 (3.583255930014657) + | > loss_mel: 20.365089416503906 (20.972548059629503) + | > loss_duration: 1.378882646560669 (1.3662329480237272) + | > amp_scaler: 256.0 (421.9680284191829) + | > loss_1: 29.4477596282959 (30.342938170763357) + | > grad_norm_1: tensor(144.9390, device='cuda:0') (tensor(139.9164, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 1.2377 (0.9241406515267137) + | > loader_time: 0.0249 (0.015333658848520289) + + + --> TIME: 2024-06-25 08:10:56 -- STEP: 588/591 -- GLOBAL_STEP: 19500 + | > loss_disc: 2.7946503162384033 (2.7313030151282836) + | > loss_disc_real_0: 0.27860769629478455 (0.21115733745197465) + | > loss_disc_real_1: 0.22218094766139984 (0.23177115207373286) + | > loss_disc_real_2: 0.23358869552612305 (0.2367343937560003) + | > loss_disc_real_3: 0.2504333257675171 (0.23592394836196284) + | > loss_disc_real_4: 0.24883539974689484 (0.23360368622099462) + | > loss_disc_real_5: 0.24634912610054016 (0.2390754455510451) + | > loss_0: 2.7946503162384033 (2.7313030151282836) + | > grad_norm_0: tensor(12.3177, device='cuda:0') (tensor(10.6480, device='cuda:0')) + | > loss_gen: 1.8019530773162842 (1.9107068730454866) + | > loss_kl: 2.422865152359009 (2.5055483591799828) + | > loss_feat: 3.2786707878112793 (3.571791627374637) + | > loss_mel: 20.020601272583008 (20.951275588703805) + | > loss_duration: 1.4130686521530151 (1.368075024108498) + | > amp_scaler: 256.0 (414.9115646258502) + | > loss_1: 28.937158584594727 (30.307397479102725) + | > grad_norm_1: tensor(71.3997, device='cuda:0') (tensor(138.2947, device='cuda:0')) + | > current_lr_0: 0.00019920154806425444 + | > current_lr_1: 0.00019920154806425444 + | > step_time: 1.2104 (0.9356819156075822) + | > loader_time: 0.0227 (0.015717666570832137) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036619977748140378 (+6.665574743392952e-06) + | > avg_loss_disc: 2.7319433334025924 (+0.04051627504064603) + | > avg_loss_disc_real_0: 0.1408236224917655 (-0.061033718922036795) + | > avg_loss_disc_real_1: 0.2427095631335644 (+0.01726223623498957) + | > avg_loss_disc_real_2: 0.23892699911239298 (+0.0023575482216287025) + | > avg_loss_disc_real_3: 0.2038699705550011 (-0.08156309958468078) + | > avg_loss_disc_real_4: 0.22702138316124043 (-0.028151686204240745) + | > avg_loss_disc_real_5: 0.2519651236052209 (+0.007357889350424446) + | > avg_loss_0: 2.7319433334025924 (+0.04051627504064603) + | > avg_loss_gen: 1.7491209709897955 (-0.21673489885127295) + | > avg_loss_kl: 2.580264279182921 (+0.06309052223854916) + | > avg_loss_feat: 3.501207544448528 (+0.03729517409142025) + | > avg_loss_mel: 20.847517581696206 (+0.07459494408140799) + | > avg_loss_duration: 1.3765274742816354 (-0.012660576942118862) + | > avg_loss_1: 30.054637908935547 (-0.05441470856362329) + + + > EPOCH: 34/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 08:11:08)  + + --> TIME: 2024-06-25 08:11:27 -- STEP: 22/591 -- GLOBAL_STEP: 19525 + | > loss_disc: 2.7577590942382812 (2.7229590957814995) + | > loss_disc_real_0: 0.15704447031021118 (0.21501695872707802) + | > loss_disc_real_1: 0.21588310599327087 (0.22549385238777508) + | > loss_disc_real_2: 0.2256346344947815 (0.23368755253878506) + | > loss_disc_real_3: 0.28194659948349 (0.23695724118839612) + | > loss_disc_real_4: 0.267068088054657 (0.23463306711478668) + | > loss_disc_real_5: 0.2816566526889801 (0.23985519192435525) + | > loss_0: 2.7577590942382812 (2.7229590957814995) + | > grad_norm_0: tensor(14.0311, device='cuda:0') (tensor(13.5722, device='cuda:0')) + | > loss_gen: 1.9534401893615723 (1.9318250255151228) + | > loss_kl: 2.6830382347106934 (2.6587060256437822) + | > loss_feat: 3.680528163909912 (3.823634765364907) + | > loss_mel: 21.286840438842773 (21.071241118691184) + | > loss_duration: 1.3205509185791016 (1.3303145441141995) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.924396514892578 (30.815721598538484) + | > grad_norm_1: tensor(183.7523, device='cuda:0') (tensor(121.9870, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.7914 (0.7725510380484841) + | > loader_time: 0.0095 (0.008567528291182085) + + + --> TIME: 2024-06-25 08:11:48 -- STEP: 47/591 -- GLOBAL_STEP: 19550 + | > loss_disc: 2.6771395206451416 (2.7147328397060955) + | > loss_disc_real_0: 0.255434513092041 (0.21271940415844004) + | > loss_disc_real_1: 0.19288216531276703 (0.22352833982477796) + | > loss_disc_real_2: 0.2310868203639984 (0.2336348295211792) + | > loss_disc_real_3: 0.2498832643032074 (0.23568026182499338) + | > loss_disc_real_4: 0.22413352131843567 (0.2340088965410882) + | > loss_disc_real_5: 0.24014078080654144 (0.2391382969440298) + | > loss_0: 2.6771395206451416 (2.7147328397060955) + | > grad_norm_0: tensor(10.0864, device='cuda:0') (tensor(10.8554, device='cuda:0')) + | > loss_gen: 1.927539348602295 (1.917323574106744) + | > loss_kl: 2.6622180938720703 (2.618626097415356) + | > loss_feat: 3.86201548576355 (3.750112406750943) + | > loss_mel: 21.130094528198242 (20.995706314736225) + | > loss_duration: 1.3487427234649658 (1.3387848519264387) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.93061065673828 (30.620553442772398) + | > grad_norm_1: tensor(91.8434, device='cuda:0') (tensor(108.9770, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.8229 (0.7880473897812215) + | > loader_time: 0.0106 (0.009222279203698992) + + + --> TIME: 2024-06-25 08:12:08 -- STEP: 72/591 -- GLOBAL_STEP: 19575 + | > loss_disc: 2.7687621116638184 (2.7178449862533136) + | > loss_disc_real_0: 0.1814001202583313 (0.21181478817015886) + | > loss_disc_real_1: 0.21420587599277496 (0.22497462357083955) + | > loss_disc_real_2: 0.21527911722660065 (0.23383615041772524) + | > loss_disc_real_3: 0.20867586135864258 (0.23525285493168566) + | > loss_disc_real_4: 0.23283672332763672 (0.2341304092357556) + | > loss_disc_real_5: 0.2314160317182541 (0.23864262799421945) + | > loss_0: 2.7687621116638184 (2.7178449862533136) + | > grad_norm_0: tensor(5.9642, device='cuda:0') (tensor(10.0226, device='cuda:0')) + | > loss_gen: 2.00000262260437 (1.915175153149499) + | > loss_kl: 2.4578306674957275 (2.594324800703261) + | > loss_feat: 3.800382137298584 (3.7271737456321716) + | > loss_mel: 21.002622604370117 (20.976410097546047) + | > loss_duration: 1.3456876277923584 (1.3449071364270317) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.606525421142578 (30.557991133795845) + | > grad_norm_1: tensor(149.1519, device='cuda:0') (tensor(141.3218, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.833 (0.7976299160056644) + | > loader_time: 0.0101 (0.009585963355170361) + + + --> TIME: 2024-06-25 08:12:30 -- STEP: 97/591 -- GLOBAL_STEP: 19600 + | > loss_disc: 2.7274084091186523 (2.722882044684026) + | > loss_disc_real_0: 0.18801239132881165 (0.2138542171145223) + | > loss_disc_real_1: 0.22375498712062836 (0.2260943368538139) + | > loss_disc_real_2: 0.2221541702747345 (0.23454525436937196) + | > loss_disc_real_3: 0.19905810058116913 (0.23557663378641777) + | > loss_disc_real_4: 0.2548866868019104 (0.2341344006282767) + | > loss_disc_real_5: 0.2655712366104126 (0.23874471743696743) + | > loss_0: 2.7274084091186523 (2.722882044684026) + | > grad_norm_0: tensor(8.4514, device='cuda:0') (tensor(11.4459, device='cuda:0')) + | > loss_gen: 1.828972578048706 (1.9192867131577325) + | > loss_kl: 2.49619460105896 (2.582381179652263) + | > loss_feat: 3.7737584114074707 (3.72783138579929) + | > loss_mel: 21.04804039001465 (21.01276216801908) + | > loss_duration: 1.3571693897247314 (1.3530578773046273) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.504135131835938 (30.59531953162754) + | > grad_norm_1: tensor(229.5896, device='cuda:0') (tensor(142.7414, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.8578 (0.8055688794126216) + | > loader_time: 0.0104 (0.009876150445839798) + + + --> TIME: 2024-06-25 08:12:51 -- STEP: 122/591 -- GLOBAL_STEP: 19625 + | > loss_disc: 2.7357146739959717 (2.7244674964029274) + | > loss_disc_real_0: 0.13824692368507385 (0.21276115216925495) + | > loss_disc_real_1: 0.2365354299545288 (0.22705261433710816) + | > loss_disc_real_2: 0.2130206674337387 (0.23471306068975417) + | > loss_disc_real_3: 0.21979385614395142 (0.2357901973558254) + | > loss_disc_real_4: 0.23213861882686615 (0.23353226690507325) + | > loss_disc_real_5: 0.21691307425498962 (0.23826599340946947) + | > loss_0: 2.7357146739959717 (2.7244674964029274) + | > grad_norm_0: tensor(15.1463, device='cuda:0') (tensor(11.2233, device='cuda:0')) + | > loss_gen: 1.847878098487854 (1.9111151597538933) + | > loss_kl: 2.485018014907837 (2.572076545387018) + | > loss_feat: 3.667444944381714 (3.696234372795605) + | > loss_mel: 21.11052131652832 (20.999578163272037) + | > loss_duration: 1.3676180839538574 (1.3544382912213684) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.47848129272461 (30.533442669227476) + | > grad_norm_1: tensor(186.2850, device='cuda:0') (tensor(139.7073, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.8426 (0.8127027437335155) + | > loader_time: 0.0113 (0.010120841323352255) + + + --> TIME: 2024-06-25 08:13:12 -- STEP: 147/591 -- GLOBAL_STEP: 19650 + | > loss_disc: 2.7486488819122314 (2.719914277394612) + | > loss_disc_real_0: 0.16585353016853333 (0.21071116499552106) + | > loss_disc_real_1: 0.23112843930721283 (0.22729470332463583) + | > loss_disc_real_2: 0.25715482234954834 (0.23476329091049375) + | > loss_disc_real_3: 0.18276354670524597 (0.23546244933897134) + | > loss_disc_real_4: 0.2624690532684326 (0.23344710114456357) + | > loss_disc_real_5: 0.2433542013168335 (0.2381255398599469) + | > loss_0: 2.7486488819122314 (2.719914277394612) + | > grad_norm_0: tensor(9.2489, device='cuda:0') (tensor(10.3885, device='cuda:0')) + | > loss_gen: 2.011627197265625 (1.9131945452722563) + | > loss_kl: 2.4336135387420654 (2.558426222833646) + | > loss_feat: 3.4571590423583984 (3.6992356793410113) + | > loss_mel: 20.715808868408203 (20.979046828081806) + | > loss_duration: 1.331479549407959 (1.3539303768248787) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.949689865112305 (30.503833770751953) + | > grad_norm_1: tensor(46.5412, device='cuda:0') (tensor(133.0172, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.8433 (0.8183553656753227) + | > loader_time: 0.0118 (0.010380671948802718) + + + --> TIME: 2024-06-25 08:13:34 -- STEP: 172/591 -- GLOBAL_STEP: 19675 + | > loss_disc: 2.6780567169189453 (2.722778074963147) + | > loss_disc_real_0: 0.22139087319374084 (0.21120656805849347) + | > loss_disc_real_1: 0.24179929494857788 (0.22757915442073068) + | > loss_disc_real_2: 0.22911258041858673 (0.23564886310419372) + | > loss_disc_real_3: 0.1902570128440857 (0.23576191007051356) + | > loss_disc_real_4: 0.24079592525959015 (0.23333191421142843) + | > loss_disc_real_5: 0.2524740397930145 (0.23821060220862544) + | > loss_0: 2.6780567169189453 (2.722778074963147) + | > grad_norm_0: tensor(6.3156, device='cuda:0') (tensor(10.2593, device='cuda:0')) + | > loss_gen: 1.893276333808899 (1.9107557181702104) + | > loss_kl: 2.5312654972076416 (2.553694668204285) + | > loss_feat: 3.601982831954956 (3.685810703177785) + | > loss_mel: 21.68210792541504 (20.983004935952106) + | > loss_duration: 1.3552610874176025 (1.3522431289040766) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.063894271850586 (30.485509251439293) + | > grad_norm_1: tensor(171.7068, device='cuda:0') (tensor(131.7413, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.8642 (0.8244954500087472) + | > loader_time: 0.0119 (0.010670216970665506) + + + --> TIME: 2024-06-25 08:13:56 -- STEP: 197/591 -- GLOBAL_STEP: 19700 + | > loss_disc: 2.6971073150634766 (2.722219353399906) + | > loss_disc_real_0: 0.10622764378786087 (0.21099035404055247) + | > loss_disc_real_1: 0.2187877744436264 (0.22697263449279184) + | > loss_disc_real_2: 0.23959393799304962 (0.23552666249008952) + | > loss_disc_real_3: 0.21386751532554626 (0.2355510130297714) + | > loss_disc_real_4: 0.20636025071144104 (0.2334714049312669) + | > loss_disc_real_5: 0.20009666681289673 (0.23833100323749679) + | > loss_0: 2.6971073150634766 (2.722219353399906) + | > grad_norm_0: tensor(29.4351, device='cuda:0') (tensor(10.5100, device='cuda:0')) + | > loss_gen: 2.040100574493408 (1.9100914291924027) + | > loss_kl: 2.5606467723846436 (2.549769330145744) + | > loss_feat: 3.8566830158233643 (3.683947103277681) + | > loss_mel: 20.879348754882812 (20.990494558653836) + | > loss_duration: 1.3520801067352295 (1.351758318504101) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.688858032226562 (30.48606083477814) + | > grad_norm_1: tensor(207.3370, device='cuda:0') (tensor(132.8892, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.8779 (0.8297470075829985) + | > loader_time: 0.0125 (0.010936214234018079) + + + --> TIME: 2024-06-25 08:14:19 -- STEP: 222/591 -- GLOBAL_STEP: 19725 + | > loss_disc: 2.7256741523742676 (2.7203898526526786) + | > loss_disc_real_0: 0.16225944459438324 (0.2107838895675298) + | > loss_disc_real_1: 0.2047160118818283 (0.2267742010655704) + | > loss_disc_real_2: 0.2330293208360672 (0.23530422869297835) + | > loss_disc_real_3: 0.26314133405685425 (0.2356713293371974) + | > loss_disc_real_4: 0.23982708156108856 (0.23325319393529548) + | > loss_disc_real_5: 0.2551248073577881 (0.23839193871160885) + | > loss_0: 2.7256741523742676 (2.7203898526526786) + | > grad_norm_0: tensor(12.8372, device='cuda:0') (tensor(10.3171, device='cuda:0')) + | > loss_gen: 1.9683828353881836 (1.9089690505921304) + | > loss_kl: 2.433073043823242 (2.5439762800663437) + | > loss_feat: 3.4319636821746826 (3.6794875065485635) + | > loss_mel: 20.246562957763672 (20.967072366594177) + | > loss_duration: 1.3486436605453491 (1.350839504787514) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.428627014160156 (30.450344798801183) + | > grad_norm_1: tensor(80.5454, device='cuda:0') (tensor(130.2385, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.8733 (0.8353133362692755) + | > loader_time: 0.0147 (0.011242072861473835) + + + --> TIME: 2024-06-25 08:14:42 -- STEP: 247/591 -- GLOBAL_STEP: 19750 + | > loss_disc: 2.7615530490875244 (2.7179580140210358) + | > loss_disc_real_0: 0.2150973379611969 (0.2107312987569855) + | > loss_disc_real_1: 0.2564481496810913 (0.22669964926204217) + | > loss_disc_real_2: 0.25832220911979675 (0.23495597739210014) + | > loss_disc_real_3: 0.29934725165367126 (0.2354827038792946) + | > loss_disc_real_4: 0.23966450989246368 (0.23308790043780678) + | > loss_disc_real_5: 0.24283155798912048 (0.23841714225558616) + | > loss_0: 2.7615530490875244 (2.7179580140210358) + | > grad_norm_0: tensor(4.9555, device='cuda:0') (tensor(10.1608, device='cuda:0')) + | > loss_gen: 1.7434839010238647 (1.9097423032227798) + | > loss_kl: 2.4631030559539795 (2.5431788643362063) + | > loss_feat: 3.502880573272705 (3.6866932629573683) + | > loss_mel: 20.505720138549805 (20.966371080653374) + | > loss_duration: 1.3692591190338135 (1.3511732724996715) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.58444595336914 (30.457158864751037) + | > grad_norm_1: tensor(117.9724, device='cuda:0') (tensor(131.7398, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.9136 (0.8407265985542945) + | > loader_time: 0.0155 (0.011580604290672638) + + + --> TIME: 2024-06-25 08:15:05 -- STEP: 272/591 -- GLOBAL_STEP: 19775 + | > loss_disc: 2.6309306621551514 (2.718163634924327) + | > loss_disc_real_0: 0.22685003280639648 (0.21093908497406277) + | > loss_disc_real_1: 0.24598141014575958 (0.2264898956808097) + | > loss_disc_real_2: 0.20005854964256287 (0.23479511983254375) + | > loss_disc_real_3: 0.19692577421665192 (0.23518051568637877) + | > loss_disc_real_4: 0.2156001329421997 (0.23318568896502254) + | > loss_disc_real_5: 0.205809086561203 (0.23848780293894165) + | > loss_0: 2.6309306621551514 (2.718163634924327) + | > grad_norm_0: tensor(6.1768, device='cuda:0') (tensor(10.3683, device='cuda:0')) + | > loss_gen: 1.830485224723816 (1.9080761925262564) + | > loss_kl: 2.5529699325561523 (2.5406858605497016) + | > loss_feat: 3.6126708984375 (3.6798050140633296) + | > loss_mel: 20.905977249145508 (20.94853837349834) + | > loss_duration: 1.3759689331054688 (1.3514934699325) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.2780704498291 (30.4285989929648) + | > grad_norm_1: tensor(163.8971, device='cuda:0') (tensor(132.0892, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.914 (0.8459527255857693) + | > loader_time: 0.013 (0.011862710118293759) + + + --> TIME: 2024-06-25 08:15:28 -- STEP: 297/591 -- GLOBAL_STEP: 19800 + | > loss_disc: 2.694666862487793 (2.7170377267330177) + | > loss_disc_real_0: 0.18211032450199127 (0.21064537550363477) + | > loss_disc_real_1: 0.23271343111991882 (0.22639309572250355) + | > loss_disc_real_2: 0.22762073576450348 (0.2348347868180837) + | > loss_disc_real_3: 0.23728376626968384 (0.23522743908845214) + | > loss_disc_real_4: 0.21105706691741943 (0.23310939596357572) + | > loss_disc_real_5: 0.2159413993358612 (0.23815494519893568) + | > loss_0: 2.694666862487793 (2.7170377267330177) + | > grad_norm_0: tensor(4.9193, device='cuda:0') (tensor(10.1777, device='cuda:0')) + | > loss_gen: 2.066420316696167 (1.9095232462642169) + | > loss_kl: 2.5706255435943604 (2.5401370629718403) + | > loss_feat: 3.80953311920166 (3.6786015683954405) + | > loss_mel: 20.802026748657227 (20.938417980566562) + | > loss_duration: 1.326249599456787 (1.3522040960363269) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.57485580444336 (30.418884065416123) + | > grad_norm_1: tensor(60.5498, device='cuda:0') (tensor(128.5867, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.9394 (0.8512257871403036) + | > loader_time: 0.0138 (0.0121036258209434) + + + --> TIME: 2024-06-25 08:15:51 -- STEP: 322/591 -- GLOBAL_STEP: 19825 + | > loss_disc: 2.738691806793213 (2.717890369225732) + | > loss_disc_real_0: 0.18289914727210999 (0.21071437019739092) + | > loss_disc_real_1: 0.2104177474975586 (0.22644310666722542) + | > loss_disc_real_2: 0.22165237367153168 (0.23514231863599386) + | > loss_disc_real_3: 0.23148241639137268 (0.2350517134673847) + | > loss_disc_real_4: 0.2277367115020752 (0.2330416689470688) + | > loss_disc_real_5: 0.20036108791828156 (0.23824417503168865) + | > loss_0: 2.738691806793213 (2.717890369225732) + | > grad_norm_0: tensor(6.3441, device='cuda:0') (tensor(10.4013, device='cuda:0')) + | > loss_gen: 2.018695116043091 (1.910375840545441) + | > loss_kl: 2.5898854732513428 (2.538134308335204) + | > loss_feat: 3.9926092624664307 (3.6764159920793134) + | > loss_mel: 21.04245376586914 (20.932546627447465) + | > loss_duration: 1.3104429244995117 (1.352952890144372) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.954086303710938 (30.410425748884307) + | > grad_norm_1: tensor(63.3192, device='cuda:0') (tensor(126.8877, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.944 (0.8570576202795372) + | > loader_time: 0.0153 (0.012343449622207547) + + + --> TIME: 2024-06-25 08:16:15 -- STEP: 347/591 -- GLOBAL_STEP: 19850 + | > loss_disc: 2.7835021018981934 (2.7190499003407598) + | > loss_disc_real_0: 0.18393653631210327 (0.2108264640617783) + | > loss_disc_real_1: 0.2445002943277359 (0.2268518287613344) + | > loss_disc_real_2: 0.2309289127588272 (0.23530567851808643) + | > loss_disc_real_3: 0.26368361711502075 (0.23526663640218784) + | > loss_disc_real_4: 0.22195087373256683 (0.23308632172837382) + | > loss_disc_real_5: 0.2323535531759262 (0.23836877332297113) + | > loss_0: 2.7835021018981934 (2.7190499003407598) + | > grad_norm_0: tensor(6.5945, device='cuda:0') (tensor(10.3641, device='cuda:0')) + | > loss_gen: 1.8486876487731934 (1.9098801083798367) + | > loss_kl: 2.4762187004089355 (2.535900511727898) + | > loss_feat: 3.4778270721435547 (3.669869212664513) + | > loss_mel: 21.349529266357422 (20.935765450557298) + | > loss_duration: 1.3632714748382568 (1.3543294812485527) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.515535354614258 (30.40574485492981) + | > grad_norm_1: tensor(65.9176, device='cuda:0') (tensor(125.4817, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.9266 (0.8630985870141461) + | > loader_time: 0.0187 (0.012588774436488962) + + + --> TIME: 2024-06-25 08:16:40 -- STEP: 372/591 -- GLOBAL_STEP: 19875 + | > loss_disc: 2.7031729221343994 (2.719569271610629) + | > loss_disc_real_0: 0.12427496910095215 (0.2108684690488923) + | > loss_disc_real_1: 0.24267494678497314 (0.2268835334607991) + | > loss_disc_real_2: 0.23852887749671936 (0.2355016991496086) + | > loss_disc_real_3: 0.2588955760002136 (0.23519868371627664) + | > loss_disc_real_4: 0.23972059786319733 (0.23315433528955265) + | > loss_disc_real_5: 0.24181008338928223 (0.23840694830462497) + | > loss_0: 2.7031729221343994 (2.719569271610629) + | > grad_norm_0: tensor(14.2748, device='cuda:0') (tensor(10.3764, device='cuda:0')) + | > loss_gen: 1.922244906425476 (1.9092052505221417) + | > loss_kl: 2.355282783508301 (2.5318272568846267) + | > loss_feat: 3.5849404335021973 (3.6638591956066824) + | > loss_mel: 20.45599365234375 (20.925102095450118) + | > loss_duration: 1.383033275604248 (1.3559234504417699) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.701496124267578 (30.385917330300938) + | > grad_norm_1: tensor(58.0554, device='cuda:0') (tensor(125.9458, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.9409 (0.8687388544441551) + | > loader_time: 0.0153 (0.012832807597293648) + + + --> TIME: 2024-06-25 08:17:05 -- STEP: 397/591 -- GLOBAL_STEP: 19900 + | > loss_disc: 2.749075412750244 (2.718764675654452) + | > loss_disc_real_0: 0.19089584052562714 (0.21081795094896025) + | > loss_disc_real_1: 0.1851625293493271 (0.22667446173258332) + | > loss_disc_real_2: 0.2400871068239212 (0.23542573323628163) + | > loss_disc_real_3: 0.23390232026576996 (0.23515200066926797) + | > loss_disc_real_4: 0.2270270735025406 (0.23290336075747944) + | > loss_disc_real_5: 0.2170676738023758 (0.23831961973938592) + | > loss_0: 2.749075412750244 (2.718764675654452) + | > grad_norm_0: tensor(7.6695, device='cuda:0') (tensor(10.2695, device='cuda:0')) + | > loss_gen: 1.9330289363861084 (1.9094091797955992) + | > loss_kl: 2.3494369983673096 (2.529587698223009) + | > loss_feat: 3.7941179275512695 (3.6602075214049674) + | > loss_mel: 21.11358642578125 (20.92443891376332) + | > loss_duration: 1.3758295774459839 (1.3569814017797892) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.56599998474121 (30.380624785531378) + | > grad_norm_1: tensor(208.7289, device='cuda:0') (tensor(127.6001, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.98 (0.8754355613170403) + | > loader_time: 0.0177 (0.013119954606447772) + + + --> TIME: 2024-06-25 08:17:30 -- STEP: 422/591 -- GLOBAL_STEP: 19925 + | > loss_disc: 2.679614543914795 (2.7184401688417554) + | > loss_disc_real_0: 0.22577276825904846 (0.21063939891579028) + | > loss_disc_real_1: 0.21143609285354614 (0.22681216369449245) + | > loss_disc_real_2: 0.24680550396442413 (0.2353650205437606) + | > loss_disc_real_3: 0.22857198119163513 (0.23516599632693694) + | > loss_disc_real_4: 0.23875415325164795 (0.23292883799821845) + | > loss_disc_real_5: 0.22453367710113525 (0.23842931415232438) + | > loss_0: 2.679614543914795 (2.7184401688417554) + | > grad_norm_0: tensor(7.2122, device='cuda:0') (tensor(10.1882, device='cuda:0')) + | > loss_gen: 1.8754383325576782 (1.9098427162916174) + | > loss_kl: 2.541001558303833 (2.5253921504269288) + | > loss_feat: 3.684974193572998 (3.65439659667806) + | > loss_mel: 21.081682205200195 (20.921902191017477) + | > loss_duration: 1.3760759830474854 (1.357747114100162) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.559173583984375 (30.369280833203646) + | > grad_norm_1: tensor(268.7527, device='cuda:0') (tensor(131.0472, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 0.9737 (0.881698798794317) + | > loader_time: 0.0183 (0.013385685134273006) + + + --> TIME: 2024-06-25 08:17:55 -- STEP: 447/591 -- GLOBAL_STEP: 19950 + | > loss_disc: 2.6639678478240967 (2.719317267138419) + | > loss_disc_real_0: 0.27706509828567505 (0.21090423950812962) + | > loss_disc_real_1: 0.2263794243335724 (0.2269411012363647) + | > loss_disc_real_2: 0.24939614534378052 (0.23542528291143294) + | > loss_disc_real_3: 0.23363789916038513 (0.23508015218330444) + | > loss_disc_real_4: 0.23668275773525238 (0.23293604700090634) + | > loss_disc_real_5: 0.22187650203704834 (0.238426606460439) + | > loss_0: 2.6639678478240967 (2.719317267138419) + | > grad_norm_0: tensor(14.2985, device='cuda:0') (tensor(10.2642, device='cuda:0')) + | > loss_gen: 1.8752893209457397 (1.909508737141654) + | > loss_kl: 2.6620795726776123 (2.5230214254434755) + | > loss_feat: 3.516352415084839 (3.647702915289791) + | > loss_mel: 20.82218360900879 (20.91830503220526) + | > loss_duration: 1.384033203125 (1.3590830071127116) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.259939193725586 (30.357621169463634) + | > grad_norm_1: tensor(270.6742, device='cuda:0') (tensor(133.0795, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 1.033 (0.8884971472507622) + | > loader_time: 0.015 (0.013639633020831968) + + + --> TIME: 2024-06-25 08:18:22 -- STEP: 472/591 -- GLOBAL_STEP: 19975 + | > loss_disc: 2.6969165802001953 (2.7193492171117817) + | > loss_disc_real_0: 0.1939643770456314 (0.21067727030396965) + | > loss_disc_real_1: 0.2466675490140915 (0.22699033342680688) + | > loss_disc_real_2: 0.2370349019765854 (0.23536255520026564) + | > loss_disc_real_3: 0.22903752326965332 (0.23519223858996974) + | > loss_disc_real_4: 0.2231433391571045 (0.23290111449689177) + | > loss_disc_real_5: 0.20749497413635254 (0.23844777009749818) + | > loss_0: 2.6969165802001953 (2.7193492171117817) + | > grad_norm_0: tensor(4.6864, device='cuda:0') (tensor(10.3677, device='cuda:0')) + | > loss_gen: 1.9573242664337158 (1.9096085403935383) + | > loss_kl: 2.5660932064056396 (2.519939199342567) + | > loss_feat: 3.5645389556884766 (3.6420874499668505) + | > loss_mel: 20.510183334350586 (20.904196820016633) + | > loss_duration: 1.3866913318634033 (1.360359220939167) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.984832763671875 (30.336191274351993) + | > grad_norm_1: tensor(43.1837, device='cuda:0') (tensor(133.3282, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 1.0078 (0.8957758485260656) + | > loader_time: 0.0199 (0.013947792982651015) + + + --> TIME: 2024-06-25 08:18:48 -- STEP: 497/591 -- GLOBAL_STEP: 20000 + | > loss_disc: 2.7254228591918945 (2.719488425034153) + | > loss_disc_real_0: 0.2547575533390045 (0.21072331276278142) + | > loss_disc_real_1: 0.20043332874774933 (0.22694236729706316) + | > loss_disc_real_2: 0.2649953365325928 (0.2354193541785361) + | > loss_disc_real_3: 0.2505369782447815 (0.2351400121836595) + | > loss_disc_real_4: 0.24174486100673676 (0.2329757564384453) + | > loss_disc_real_5: 0.27733340859413147 (0.238574457600323) + | > loss_0: 2.7254228591918945 (2.719488425034153) + | > grad_norm_0: tensor(7.8112, device='cuda:0') (tensor(10.3212, device='cuda:0')) + | > loss_gen: 1.8281161785125732 (1.9089018648539031) + | > loss_kl: 2.476038932800293 (2.5181353202528167) + | > loss_feat: 3.4979121685028076 (3.635984198668113) + | > loss_mel: 19.991554260253906 (20.884734746675857) + | > loss_duration: 1.3963537216186523 (1.3615973888028792) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.18997573852539 (30.309353546357492) + | > grad_norm_1: tensor(59.0592, device='cuda:0') (tensor(131.4991, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 1.0943 (0.9031224121270285) + | > loader_time: 0.0212 (0.014243389279309893) + + + > CHECKPOINT : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/checkpoint_20000.pth + + --> TIME: 2024-06-25 08:19:20 -- STEP: 522/591 -- GLOBAL_STEP: 20025 + | > loss_disc: 2.7829620838165283 (2.7199112505748353) + | > loss_disc_real_0: 0.3056379556655884 (0.2105793669334545) + | > loss_disc_real_1: 0.2600056529045105 (0.22711165032395914) + | > loss_disc_real_2: 0.2789607048034668 (0.23567880398925695) + | > loss_disc_real_3: 0.25605592131614685 (0.2352624775063946) + | > loss_disc_real_4: 0.25431182980537415 (0.2329237581840877) + | > loss_disc_real_5: 0.25360244512557983 (0.23855016914364022) + | > loss_0: 2.7829620838165283 (2.7199112505748353) + | > grad_norm_0: tensor(16.0188, device='cuda:0') (tensor(10.3421, device='cuda:0')) + | > loss_gen: 1.7958492040634155 (1.9091725264929265) + | > loss_kl: 2.484839677810669 (2.517907399327363) + | > loss_feat: 3.321878433227539 (3.6335390527586364) + | > loss_mel: 19.909364700317383 (20.879125902022434) + | > loss_duration: 1.3987855911254883 (1.363201535524536) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.910717010498047 (30.302946441475004) + | > grad_norm_1: tensor(135.5243, device='cuda:0') (tensor(130.0232, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 1.0978 (0.9120761023627387) + | > loader_time: 0.0247 (0.014560881702379248) + + + --> TIME: 2024-06-25 08:19:49 -- STEP: 547/591 -- GLOBAL_STEP: 20050 + | > loss_disc: 2.7765421867370605 (2.72075754066272) + | > loss_disc_real_0: 0.13402722775936127 (0.21052604871487052) + | > loss_disc_real_1: 0.2484818547964096 (0.2271357724609933) + | > loss_disc_real_2: 0.2228752225637436 (0.23580189915527155) + | > loss_disc_real_3: 0.2547644376754761 (0.2352043345870222) + | > loss_disc_real_4: 0.2338741421699524 (0.23308711990686834) + | > loss_disc_real_5: 0.21936318278312683 (0.23864315542812975) + | > loss_0: 2.7765421867370605 (2.72075754066272) + | > grad_norm_0: tensor(17.0868, device='cuda:0') (tensor(10.4110, device='cuda:0')) + | > loss_gen: 1.8650375604629517 (1.9076312884988036) + | > loss_kl: 2.563249349594116 (2.5151298412239322) + | > loss_feat: 3.4965031147003174 (3.6237190985810384) + | > loss_mel: 20.36009979248047 (20.863724661483626) + | > loss_duration: 1.4136098623275757 (1.3648181447599228) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.698501586914062 (30.27502305416128) + | > grad_norm_1: tensor(98.2919, device='cuda:0') (tensor(129.0573, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 1.1358 (0.9234304009669442) + | > loader_time: 0.0239 (0.014917122162674202) + + + --> TIME: 2024-06-25 08:20:20 -- STEP: 572/591 -- GLOBAL_STEP: 20075 + | > loss_disc: 2.7595622539520264 (2.7213034884079357) + | > loss_disc_real_0: 0.20032569766044617 (0.210551555509438) + | > loss_disc_real_1: 0.19910065829753876 (0.22730407697129082) + | > loss_disc_real_2: 0.23176759481430054 (0.23577513761974714) + | > loss_disc_real_3: 0.2367534339427948 (0.23528917688694034) + | > loss_disc_real_4: 0.2354513555765152 (0.23319812124217307) + | > loss_disc_real_5: 0.25260481238365173 (0.2387986659899458) + | > loss_0: 2.7595622539520264 (2.7213034884079357) + | > grad_norm_0: tensor(3.5879, device='cuda:0') (tensor(10.3166, device='cuda:0')) + | > loss_gen: 1.7992117404937744 (1.906745778513955) + | > loss_kl: 2.4409799575805664 (2.512445274349694) + | > loss_feat: 3.3123185634613037 (3.6144974615190404) + | > loss_mel: 20.972030639648438 (20.85295452438035) + | > loss_duration: 1.4044740200042725 (1.366450148445743) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.92901611328125 (30.25309319929643) + | > grad_norm_1: tensor(104.8483, device='cuda:0') (tensor(129.6527, device='cuda:0')) + | > current_lr_0: 0.0001991766478707464 + | > current_lr_1: 0.0001991766478707464 + | > step_time: 1.3373 (0.9362965768033807) + | > loader_time: 0.0223 (0.015314568172801624) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003791981555045919 (+0.00012998378023188146) + | > avg_loss_disc: 2.731777160725695 (-0.00016617267689733595) + | > avg_loss_disc_real_0: 0.26617455831233494 (+0.12535093582056944) + | > avg_loss_disc_real_1: 0.17738339812197584 (-0.06532616501158856) + | > avg_loss_disc_real_2: 0.2504524889778583 (+0.011525489865465338) + | > avg_loss_disc_real_3: 0.26882190051230986 (+0.06495192995730875) + | > avg_loss_disc_real_4: 0.22083167731761932 (-0.006189705843621107) + | > avg_loss_disc_real_5: 0.265250321715436 (+0.013285198110215113) + | > avg_loss_0: 2.731777160725695 (-0.00016617267689733595) + | > avg_loss_gen: 1.9649472693179517 (+0.21582629832815625) + | > avg_loss_kl: 2.54570487712292 (-0.034559402060001077) + | > avg_loss_feat: 3.6910314712118595 (+0.18982392676333149) + | > avg_loss_mel: 20.999893918950512 (+0.15237633725430655) + | > avg_loss_duration: 1.3851879302491532 (+0.008660455967517855) + | > avg_loss_1: 30.586765411052298 (+0.5321275021167509) + + + > EPOCH: 35/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 08:20:52)  + + --> TIME: 2024-06-25 08:20:58 -- STEP: 6/591 -- GLOBAL_STEP: 20100 + | > loss_disc: 2.6765809059143066 (2.7115445534388223) + | > loss_disc_real_0: 0.2019536793231964 (0.21243921170632044) + | > loss_disc_real_1: 0.22941073775291443 (0.24091780930757523) + | > loss_disc_real_2: 0.19321218132972717 (0.21435562272866568) + | > loss_disc_real_3: 0.21541497111320496 (0.22291828443606695) + | > loss_disc_real_4: 0.22342079877853394 (0.2365450312693914) + | > loss_disc_real_5: 0.2390945702791214 (0.243019200861454) + | > loss_0: 2.6765809059143066 (2.7115445534388223) + | > grad_norm_0: tensor(3.4717, device='cuda:0') (tensor(13.3989, device='cuda:0')) + | > loss_gen: 1.7740583419799805 (1.930459717909495) + | > loss_kl: 2.6144466400146484 (2.6676278909047446) + | > loss_feat: 3.8439929485321045 (3.918853203455607) + | > loss_mel: 21.377098083496094 (21.073057174682617) + | > loss_duration: 1.3269212245941162 (1.3504917820294697) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.9365177154541 (30.940490404764812) + | > grad_norm_1: tensor(186.3038, device='cuda:0') (tensor(103.4472, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.7633 (0.7750180959701538) + | > loader_time: 0.0084 (0.008610010147094727) + + + --> TIME: 2024-06-25 08:21:18 -- STEP: 31/591 -- GLOBAL_STEP: 20125 + | > loss_disc: 2.7454521656036377 (2.7219058544405046) + | > loss_disc_real_0: 0.2027612328529358 (0.2042853029504899) + | > loss_disc_real_1: 0.25690212845802307 (0.22991645095809812) + | > loss_disc_real_2: 0.2621414363384247 (0.23918695555579278) + | > loss_disc_real_3: 0.26934143900871277 (0.2352748719915267) + | > loss_disc_real_4: 0.25588253140449524 (0.23440863240149715) + | > loss_disc_real_5: 0.2538571357727051 (0.2400086017385606) + | > loss_0: 2.7454521656036377 (2.7219058544405046) + | > grad_norm_0: tensor(4.2215, device='cuda:0') (tensor(9.2515, device='cuda:0')) + | > loss_gen: 1.9282277822494507 (1.9323226636455906) + | > loss_kl: 2.666560649871826 (2.6250348244943926) + | > loss_feat: 3.415816307067871 (3.7850199053364415) + | > loss_mel: 20.657800674438477 (20.913930400725338) + | > loss_duration: 1.3860774040222168 (1.3317615908961142) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.05448341369629 (30.588069546607233) + | > grad_norm_1: tensor(212.6280, device='cuda:0') (tensor(128.8616, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.7794 (0.7819538731728831) + | > loader_time: 0.0092 (0.009105520863686838) + + + --> TIME: 2024-06-25 08:21:39 -- STEP: 56/591 -- GLOBAL_STEP: 20150 + | > loss_disc: 2.680234670639038 (2.7289970432009016) + | > loss_disc_real_0: 0.15977275371551514 (0.2094705542549491) + | > loss_disc_real_1: 0.22981604933738708 (0.229193703936679) + | > loss_disc_real_2: 0.22099336981773376 (0.23614140360483102) + | > loss_disc_real_3: 0.26088324189186096 (0.2343903918351446) + | > loss_disc_real_4: 0.23250369727611542 (0.23468461952039174) + | > loss_disc_real_5: 0.21244321763515472 (0.23929851342524802) + | > loss_0: 2.680234670639038 (2.7289970432009016) + | > grad_norm_0: tensor(12.9652, device='cuda:0') (tensor(10.7487, device='cuda:0')) + | > loss_gen: 2.1354451179504395 (1.9194710510117667) + | > loss_kl: 2.5119707584381104 (2.5916654765605927) + | > loss_feat: 4.049454212188721 (3.7461863500731334) + | > loss_mel: 21.37520980834961 (20.90598065512521) + | > loss_duration: 1.4040664434432983 (1.3432752170733042) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.476144790649414 (30.506578751972743) + | > grad_norm_1: tensor(273.7719, device='cuda:0') (tensor(136.8559, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.8013 (0.7926062728677478) + | > loader_time: 0.0097 (0.009534746408462526) + + + --> TIME: 2024-06-25 08:22:00 -- STEP: 81/591 -- GLOBAL_STEP: 20175 + | > loss_disc: 2.73209285736084 (2.728730328289079) + | > loss_disc_real_0: 0.2374868392944336 (0.21154769427246517) + | > loss_disc_real_1: 0.23820196092128754 (0.22914967823911597) + | > loss_disc_real_2: 0.29427969455718994 (0.23628433710998958) + | > loss_disc_real_3: 0.25560182332992554 (0.23425269917941388) + | > loss_disc_real_4: 0.24857334792613983 (0.23512501223587695) + | > loss_disc_real_5: 0.26904407143592834 (0.24030758274926078) + | > loss_0: 2.73209285736084 (2.728730328289079) + | > grad_norm_0: tensor(7.7047, device='cuda:0') (tensor(10.9198, device='cuda:0')) + | > loss_gen: 1.8938714265823364 (1.916198933566058) + | > loss_kl: 2.668348789215088 (2.5864316916760104) + | > loss_feat: 3.588066339492798 (3.730564965142144) + | > loss_mel: 20.96108055114746 (20.938120971491312) + | > loss_duration: 1.3605042695999146 (1.3487362037470314) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.47187042236328 (30.520052733244718) + | > grad_norm_1: tensor(76.3222, device='cuda:0') (tensor(143.1660, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.8224 (0.8057522155620436) + | > loader_time: 0.0105 (0.009865422307709119) + + + --> TIME: 2024-06-25 08:22:21 -- STEP: 106/591 -- GLOBAL_STEP: 20200 + | > loss_disc: 2.6946136951446533 (2.724216159784568) + | > loss_disc_real_0: 0.18466857075691223 (0.209895928372752) + | > loss_disc_real_1: 0.20284606516361237 (0.228296678443) + | > loss_disc_real_2: 0.2369394600391388 (0.23594423875493822) + | > loss_disc_real_3: 0.24072405695915222 (0.23449739223381258) + | > loss_disc_real_4: 0.2326086014509201 (0.23462853327674685) + | > loss_disc_real_5: 0.26432639360427856 (0.24041592877990794) + | > loss_0: 2.6946136951446533 (2.724216159784568) + | > grad_norm_0: tensor(3.7876, device='cuda:0') (tensor(9.7834, device='cuda:0')) + | > loss_gen: 1.8705496788024902 (1.9108987371876556) + | > loss_kl: 2.423220634460449 (2.5761212740304344) + | > loss_feat: 3.730191946029663 (3.7191874756003327) + | > loss_mel: 21.042110443115234 (20.951196310655135) + | > loss_duration: 1.368370771408081 (1.3545808769621945) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.4344425201416 (30.511984609208017) + | > grad_norm_1: tensor(234.4725, device='cuda:0') (tensor(145.9803, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.8383 (0.8119591542010038) + | > loader_time: 0.0115 (0.010207533836364746) + + + --> TIME: 2024-06-25 08:22:43 -- STEP: 131/591 -- GLOBAL_STEP: 20225 + | > loss_disc: 2.701612949371338 (2.72229231586893) + | > loss_disc_real_0: 0.21572601795196533 (0.20993838487690641) + | > loss_disc_real_1: 0.23977673053741455 (0.22805320147339625) + | > loss_disc_real_2: 0.20409329235553741 (0.23542681347777825) + | > loss_disc_real_3: 0.24125918745994568 (0.2343435267000708) + | > loss_disc_real_4: 0.22171235084533691 (0.2342182360305131) + | > loss_disc_real_5: 0.239595428109169 (0.23967352566373257) + | > loss_0: 2.701612949371338 (2.72229231586893) + | > grad_norm_0: tensor(7.9322, device='cuda:0') (tensor(10.1721, device='cuda:0')) + | > loss_gen: 1.9522987604141235 (1.9144890717877687) + | > loss_kl: 2.5164377689361572 (2.5657113271815186) + | > loss_feat: 3.631726026535034 (3.7241375537319037) + | > loss_mel: 21.1289005279541 (20.984084791809554) + | > loss_duration: 1.3838553428649902 (1.355364748539816) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.61322021484375 (30.54378741024105) + | > grad_norm_1: tensor(90.9633, device='cuda:0') (tensor(141.8161, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.8669 (0.8181155219332864) + | > loader_time: 0.0115 (0.010470415799672367) + + + --> TIME: 2024-06-25 08:23:04 -- STEP: 156/591 -- GLOBAL_STEP: 20250 + | > loss_disc: 2.7379961013793945 (2.720507079210036) + | > loss_disc_real_0: 0.2877890169620514 (0.2093367753311609) + | > loss_disc_real_1: 0.24676698446273804 (0.22777825823196998) + | > loss_disc_real_2: 0.2214937061071396 (0.23587559497891328) + | > loss_disc_real_3: 0.26030659675598145 (0.23446763020295364) + | > loss_disc_real_4: 0.23090726137161255 (0.23460823459885058) + | > loss_disc_real_5: 0.22068358957767487 (0.2394594613176126) + | > loss_0: 2.7379961013793945 (2.720507079210036) + | > grad_norm_0: tensor(15.4208, device='cuda:0') (tensor(9.6628, device='cuda:0')) + | > loss_gen: 1.7855825424194336 (1.9152583143649957) + | > loss_kl: 2.5855443477630615 (2.5567323458500404) + | > loss_feat: 3.7900612354278564 (3.729393407320365) + | > loss_mel: 20.969030380249023 (20.993856112162273) + | > loss_duration: 1.33815598487854 (1.3536351399543964) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.468374252319336 (30.5488752585191) + | > grad_norm_1: tensor(169.5076, device='cuda:0') (tensor(143.2812, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.8644 (0.8241488169401122) + | > loader_time: 0.0117 (0.010750923401270155) + + + --> TIME: 2024-06-25 08:23:26 -- STEP: 181/591 -- GLOBAL_STEP: 20275 + | > loss_disc: 2.686441659927368 (2.7211738673362937) + | > loss_disc_real_0: 0.1564951241016388 (0.20926875979202225) + | > loss_disc_real_1: 0.24559207260608673 (0.2279149968979767) + | > loss_disc_real_2: 0.21389247477054596 (0.23604204855571137) + | > loss_disc_real_3: 0.2615906000137329 (0.2348752961138994) + | > loss_disc_real_4: 0.24048550426959991 (0.23432557301297371) + | > loss_disc_real_5: 0.23028862476348877 (0.23944056964381624) + | > loss_0: 2.686441659927368 (2.7211738673362937) + | > grad_norm_0: tensor(11.1542, device='cuda:0') (tensor(9.8487, device='cuda:0')) + | > loss_gen: 1.9597759246826172 (1.9168905605927358) + | > loss_kl: 2.574077606201172 (2.5486632934591382) + | > loss_feat: 3.6622462272644043 (3.7249773352185667) + | > loss_mel: 21.019041061401367 (20.995071779957136) + | > loss_duration: 1.3319096565246582 (1.3517483387204177) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.54705047607422 (30.537351239452047) + | > grad_norm_1: tensor(195.7854, device='cuda:0') (tensor(142.4145, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.8613 (0.8292314321296651) + | > loader_time: 0.0116 (0.010955616914106334) + + + --> TIME: 2024-06-25 08:23:49 -- STEP: 206/591 -- GLOBAL_STEP: 20300 + | > loss_disc: 2.6652824878692627 (2.7194064033841623) + | > loss_disc_real_0: 0.1824299693107605 (0.20962000457407198) + | > loss_disc_real_1: 0.24784402549266815 (0.2276986066288161) + | > loss_disc_real_2: 0.2553432285785675 (0.23568319919907932) + | > loss_disc_real_3: 0.2697838246822357 (0.2349179701753033) + | > loss_disc_real_4: 0.2143540233373642 (0.2337558785977873) + | > loss_disc_real_5: 0.22626054286956787 (0.23929549312417947) + | > loss_0: 2.6652824878692627 (2.7194064033841623) + | > grad_norm_0: tensor(5.4911, device='cuda:0') (tensor(9.9452, device='cuda:0')) + | > loss_gen: 1.8167327642440796 (1.9167517319466303) + | > loss_kl: 2.492469549179077 (2.539371440711531) + | > loss_feat: 3.7474749088287354 (3.72801043339146) + | > loss_mel: 20.859060287475586 (21.005302494012035) + | > loss_duration: 1.341624140739441 (1.3509390689794305) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.257360458374023 (30.54037511695936) + | > grad_norm_1: tensor(182.8976, device='cuda:0') (tensor(139.2194, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.8859 (0.8343722473070461) + | > loader_time: 0.0134 (0.01118188691370695) + + + --> TIME: 2024-06-25 08:24:11 -- STEP: 231/591 -- GLOBAL_STEP: 20325 + | > loss_disc: 2.6524670124053955 (2.7175549232598506) + | > loss_disc_real_0: 0.1538163721561432 (0.20913978727348953) + | > loss_disc_real_1: 0.18954360485076904 (0.22712999640347123) + | > loss_disc_real_2: 0.2298232913017273 (0.23536183339950842) + | > loss_disc_real_3: 0.23683595657348633 (0.23471440994120263) + | > loss_disc_real_4: 0.2148854285478592 (0.23402055394855928) + | > loss_disc_real_5: 0.23812030255794525 (0.23949212022932062) + | > loss_0: 2.6524670124053955 (2.7175549232598506) + | > grad_norm_0: tensor(9.6380, device='cuda:0') (tensor(9.8942, device='cuda:0')) + | > loss_gen: 2.0974843502044678 (1.9177395549171414) + | > loss_kl: 2.5496859550476074 (2.5345496429509415) + | > loss_feat: 3.949397325515747 (3.7249089783920355) + | > loss_mel: 21.208301544189453 (21.006642494366798) + | > loss_duration: 1.356469750404358 (1.3506221260343285) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.161340713500977 (30.53446275124818) + | > grad_norm_1: tensor(272.1022, device='cuda:0') (tensor(140.1933, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.8813 (0.8392871530541095) + | > loader_time: 0.0162 (0.011486054498911932) + + + --> TIME: 2024-06-25 08:24:34 -- STEP: 256/591 -- GLOBAL_STEP: 20350 + | > loss_disc: 2.7047617435455322 (2.7154398281127214) + | > loss_disc_real_0: 0.23141363263130188 (0.20863184327026826) + | > loss_disc_real_1: 0.28098970651626587 (0.22753280331380665) + | > loss_disc_real_2: 0.288426011800766 (0.23536671459442005) + | > loss_disc_real_3: 0.2544243037700653 (0.2347199452924542) + | > loss_disc_real_4: 0.23955929279327393 (0.23391791008180007) + | > loss_disc_real_5: 0.23747208714485168 (0.23921205673832446) + | > loss_0: 2.7047617435455322 (2.7154398281127214) + | > grad_norm_0: tensor(6.8524, device='cuda:0') (tensor(9.7540, device='cuda:0')) + | > loss_gen: 1.8585671186447144 (1.9185871831141412) + | > loss_kl: 2.4429755210876465 (2.534362940117717) + | > loss_feat: 3.637040853500366 (3.7238066233694553) + | > loss_mel: 20.94504165649414 (21.010422751307487) + | > loss_duration: 1.374185562133789 (1.3510215696878736) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.257810592651367 (30.5382010191679) + | > grad_norm_1: tensor(295.8664, device='cuda:0') (tensor(147.5017, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.8897 (0.8445274215191604) + | > loader_time: 0.0174 (0.01178792770951986) + + + --> TIME: 2024-06-25 08:24:57 -- STEP: 281/591 -- GLOBAL_STEP: 20375 + | > loss_disc: 2.765530586242676 (2.715984130669319) + | > loss_disc_real_0: 0.09127169102430344 (0.20918233928625266) + | > loss_disc_real_1: 0.2176257073879242 (0.22725820074726255) + | > loss_disc_real_2: 0.2300279289484024 (0.23516486176816595) + | > loss_disc_real_3: 0.20122061669826508 (0.23464955638736168) + | > loss_disc_real_4: 0.2040150910615921 (0.23358381144516832) + | > loss_disc_real_5: 0.2186945378780365 (0.23918066743854102) + | > loss_0: 2.765530586242676 (2.715984130669319) + | > grad_norm_0: tensor(26.2592, device='cuda:0') (tensor(10.2997, device='cuda:0')) + | > loss_gen: 1.7816354036331177 (1.91872169708442) + | > loss_kl: 2.4742255210876465 (2.533706934850835) + | > loss_feat: 3.75376296043396 (3.7258668030708284) + | > loss_mel: 21.092437744140625 (21.004182618707947) + | > loss_duration: 1.3869057893753052 (1.3517049457679022) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.488967895507812 (30.534182979543015) + | > grad_norm_1: tensor(254.7086, device='cuda:0') (tensor(147.4814, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.9095 (0.8497177358199695) + | > loader_time: 0.0155 (0.012074177799699994) + + + --> TIME: 2024-06-25 08:25:20 -- STEP: 306/591 -- GLOBAL_STEP: 20400 + | > loss_disc: 2.717805862426758 (2.714788326250961) + | > loss_disc_real_0: 0.18927140533924103 (0.20901416556215752) + | > loss_disc_real_1: 0.2257283478975296 (0.22712665830367532) + | > loss_disc_real_2: 0.23888438940048218 (0.23496778114558825) + | > loss_disc_real_3: 0.22516320645809174 (0.23468877073206934) + | > loss_disc_real_4: 0.24017077684402466 (0.23353793973626655) + | > loss_disc_real_5: 0.2505333125591278 (0.2391064362783058) + | > loss_0: 2.717805862426758 (2.714788326250961) + | > grad_norm_0: tensor(3.6506, device='cuda:0') (tensor(9.9810, device='cuda:0')) + | > loss_gen: 1.8865175247192383 (1.918400968991074) + | > loss_kl: 2.59376859664917 (2.5325142230862876) + | > loss_feat: 3.5976593494415283 (3.722767917159336) + | > loss_mel: 21.060808181762695 (20.98790524052639) + | > loss_duration: 1.382428526878357 (1.3521938518761036) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.521181106567383 (30.51378217086293) + | > grad_norm_1: tensor(177.6357, device='cuda:0') (tensor(147.1205, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.9097 (0.8551075302697475) + | > loader_time: 0.0135 (0.01231716349233989) + + + --> TIME: 2024-06-25 08:25:44 -- STEP: 331/591 -- GLOBAL_STEP: 20425 + | > loss_disc: 2.6412384510040283 (2.712894732138181) + | > loss_disc_real_0: 0.13013029098510742 (0.2088095494672611) + | > loss_disc_real_1: 0.18319463729858398 (0.2268620855048344) + | > loss_disc_real_2: 0.2273123562335968 (0.2348867216319116) + | > loss_disc_real_3: 0.2220028191804886 (0.23467522688502632) + | > loss_disc_real_4: 0.23701001703739166 (0.23324998916095838) + | > loss_disc_real_5: 0.25430941581726074 (0.23896626434837584) + | > loss_0: 2.6412384510040283 (2.712894732138181) + | > grad_norm_0: tensor(16.7611, device='cuda:0') (tensor(9.8586, device='cuda:0')) + | > loss_gen: 2.0330700874328613 (1.9194699997628204) + | > loss_kl: 2.530133008956909 (2.5315355066086225) + | > loss_feat: 4.076212406158447 (3.7236365710140356) + | > loss_mel: 20.82404136657715 (20.97204398532649) + | > loss_duration: 1.3554918766021729 (1.3529345157283315) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.818946838378906 (30.499620535582576) + | > grad_norm_1: tensor(194.1640, device='cuda:0') (tensor(147.2584, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.9375 (0.8601986555174398) + | > loader_time: 0.018 (0.01261010775033081) + + + --> TIME: 2024-06-25 08:26:08 -- STEP: 356/591 -- GLOBAL_STEP: 20450 + | > loss_disc: 2.7116446495056152 (2.7131963733876687) + | > loss_disc_real_0: 0.28855448961257935 (0.20909903887031453) + | > loss_disc_real_1: 0.2055952101945877 (0.2268182069816616) + | > loss_disc_real_2: 0.1941138058900833 (0.2350388103991412) + | > loss_disc_real_3: 0.2561325430870056 (0.23464777098780268) + | > loss_disc_real_4: 0.2240479290485382 (0.23332121767354816) + | > loss_disc_real_5: 0.2247847616672516 (0.2390115348559417) + | > loss_0: 2.7116446495056152 (2.7131963733876687) + | > grad_norm_0: tensor(16.5885, device='cuda:0') (tensor(9.9905, device='cuda:0')) + | > loss_gen: 1.8271899223327637 (1.9197509362456504) + | > loss_kl: 2.689603090286255 (2.5315198811252446) + | > loss_feat: 3.8575193881988525 (3.71970406275117) + | > loss_mel: 20.957712173461914 (20.960151318753713) + | > loss_duration: 1.3666633367538452 (1.354168344749494) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.698688507080078 (30.485294502772643) + | > grad_norm_1: tensor(39.4240, device='cuda:0') (tensor(149.1136, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.9555 (0.8662364590034056) + | > loader_time: 0.0171 (0.012859232639998535) + + + --> TIME: 2024-06-25 08:26:33 -- STEP: 381/591 -- GLOBAL_STEP: 20475 + | > loss_disc: 2.668335437774658 (2.71216723361979) + | > loss_disc_real_0: 0.12455324083566666 (0.20877285788613043) + | > loss_disc_real_1: 0.1957884132862091 (0.22679446105725495) + | > loss_disc_real_2: 0.2343181073665619 (0.2350897755366298) + | > loss_disc_real_3: 0.25652778148651123 (0.23454887832556498) + | > loss_disc_real_4: 0.2146271914243698 (0.2330516340147479) + | > loss_disc_real_5: 0.23488619923591614 (0.23887702001361397) + | > loss_0: 2.668335437774658 (2.71216723361979) + | > grad_norm_0: tensor(17.8198, device='cuda:0') (tensor(10.0981, device='cuda:0')) + | > loss_gen: 2.018735408782959 (1.9204197718402531) + | > loss_kl: 2.441433906555176 (2.5301099712141535) + | > loss_feat: 3.8396155834198 (3.7179371284374727) + | > loss_mel: 20.229297637939453 (20.94338510668499) + | > loss_duration: 1.380237102508545 (1.3555606907121156) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.909320831298828 (30.467412638226207) + | > grad_norm_1: tensor(90.2358, device='cuda:0') (tensor(147.5134, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.9838 (0.8720681273092435) + | > loader_time: 0.0175 (0.013141234715779625) + + + --> TIME: 2024-06-25 08:26:57 -- STEP: 406/591 -- GLOBAL_STEP: 20500 + | > loss_disc: 2.694753646850586 (2.7130383587823124) + | > loss_disc_real_0: 0.21706989407539368 (0.20883874354764745) + | > loss_disc_real_1: 0.1991700381040573 (0.22690216455553552) + | > loss_disc_real_2: 0.250874400138855 (0.23519072435759558) + | > loss_disc_real_3: 0.23388619720935822 (0.23471447509791463) + | > loss_disc_real_4: 0.2525594234466553 (0.2331591887133462) + | > loss_disc_real_5: 0.2286355197429657 (0.23892343183750003) + | > loss_0: 2.694753646850586 (2.7130383587823124) + | > grad_norm_0: tensor(5.6514, device='cuda:0') (tensor(10.0139, device='cuda:0')) + | > loss_gen: 1.937653660774231 (1.9193084665707179) + | > loss_kl: 2.352290630340576 (2.5283847988532684) + | > loss_feat: 3.4781880378723145 (3.708846120998777) + | > loss_mel: 20.020402908325195 (20.935601957913125) + | > loss_duration: 1.3641308546066284 (1.3565188041461516) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.152666091918945 (30.448660131745736) + | > grad_norm_1: tensor(96.6268, device='cuda:0') (tensor(145.5910, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 1.0075 (0.8784804490986716) + | > loader_time: 0.0187 (0.013406971992530261) + + + --> TIME: 2024-06-25 08:27:23 -- STEP: 431/591 -- GLOBAL_STEP: 20525 + | > loss_disc: 2.6946356296539307 (2.71367128708523) + | > loss_disc_real_0: 0.19558490812778473 (0.20885145254519588) + | > loss_disc_real_1: 0.2213185727596283 (0.22710654426478455) + | > loss_disc_real_2: 0.22157348692417145 (0.23517711094111687) + | > loss_disc_real_3: 0.21964970231056213 (0.23483241610488317) + | > loss_disc_real_4: 0.214053675532341 (0.2330748853025193) + | > loss_disc_real_5: 0.220717191696167 (0.2387948195923232) + | > loss_0: 2.6946356296539307 (2.71367128708523) + | > grad_norm_0: tensor(5.2336, device='cuda:0') (tensor(9.9313, device='cuda:0')) + | > loss_gen: 2.0827643871307373 (1.9185089332045093) + | > loss_kl: 2.417018175125122 (2.5256163559491007) + | > loss_feat: 3.8787760734558105 (3.7006800036419296) + | > loss_mel: 20.713884353637695 (20.920156952399964) + | > loss_duration: 1.3704414367675781 (1.357306304774539) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.46288299560547 (30.42226853337476) + | > grad_norm_1: tensor(107.8257, device='cuda:0') (tensor(144.2603, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 0.9844 (0.8849374921581862) + | > loader_time: 0.0177 (0.013688096314184628) + + + --> TIME: 2024-06-25 08:27:49 -- STEP: 456/591 -- GLOBAL_STEP: 20550 + | > loss_disc: 2.6860103607177734 (2.7150368863030483) + | > loss_disc_real_0: 0.19722144305706024 (0.20930523595266176) + | > loss_disc_real_1: 0.23971666395664215 (0.2271788125171473) + | > loss_disc_real_2: 0.2508015036582947 (0.23535070048743173) + | > loss_disc_real_3: 0.25442877411842346 (0.23498701186556564) + | > loss_disc_real_4: 0.23272709548473358 (0.23313724272475952) + | > loss_disc_real_5: 0.2437562346458435 (0.2390112749447948) + | > loss_0: 2.6860103607177734 (2.7150368863030483) + | > grad_norm_0: tensor(4.9403, device='cuda:0') (tensor(10.0753, device='cuda:0')) + | > loss_gen: 1.787632942199707 (1.9167791191945995) + | > loss_kl: 2.458408832550049 (2.5227056354807136) + | > loss_feat: 3.400641441345215 (3.689390633712735) + | > loss_mel: 20.734111785888672 (20.905613430759384) + | > loss_duration: 1.3451915979385376 (1.3582404744729664) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.72598648071289 (30.392729244734113) + | > grad_norm_1: tensor(188.5159, device='cuda:0') (tensor(144.3548, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 1.002 (0.8915665505225199) + | > loader_time: 0.0184 (0.013970207749751579) + + + --> TIME: 2024-06-25 08:28:15 -- STEP: 481/591 -- GLOBAL_STEP: 20575 + | > loss_disc: 2.774482488632202 (2.7155999259988386) + | > loss_disc_real_0: 0.18006817996501923 (0.2090691815047155) + | > loss_disc_real_1: 0.23647914826869965 (0.22729083942152606) + | > loss_disc_real_2: 0.2455199807882309 (0.2353460970266941) + | > loss_disc_real_3: 0.3049963712692261 (0.23527467678341696) + | > loss_disc_real_4: 0.23152770102024078 (0.2331046398860749) + | > loss_disc_real_5: 0.2509475350379944 (0.23892281596353296) + | > loss_0: 2.774482488632202 (2.7155999259988386) + | > grad_norm_0: tensor(7.2981, device='cuda:0') (tensor(10.0825, device='cuda:0')) + | > loss_gen: 1.893465518951416 (1.9160262323962427) + | > loss_kl: 2.4571831226348877 (2.5191003458415646) + | > loss_feat: 3.420776128768921 (3.679820043877108) + | > loss_mel: 20.141401290893555 (20.885769401915127) + | > loss_duration: 1.3471964597702026 (1.359594686115606) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.260021209716797 (30.360310671483155) + | > grad_norm_1: tensor(85.5291, device='cuda:0') (tensor(142.5985, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 1.0562 (0.8985153335047859) + | > loader_time: 0.0228 (0.01421438482843665) + + + --> TIME: 2024-06-25 08:28:42 -- STEP: 506/591 -- GLOBAL_STEP: 20600 + | > loss_disc: 2.6569738388061523 (2.7151143918395513) + | > loss_disc_real_0: 0.2464129775762558 (0.20878500827098553) + | > loss_disc_real_1: 0.21197901666164398 (0.22730148686721863) + | > loss_disc_real_2: 0.25798988342285156 (0.23532553043643475) + | > loss_disc_real_3: 0.2557447850704193 (0.2351497879671485) + | > loss_disc_real_4: 0.22742509841918945 (0.23320683480604834) + | > loss_disc_real_5: 0.23174001276493073 (0.2389740654720148) + | > loss_0: 2.6569738388061523 (2.7151143918395513) + | > grad_norm_0: tensor(8.3872, device='cuda:0') (tensor(10.1354, device='cuda:0')) + | > loss_gen: 1.9942373037338257 (1.9158225681470789) + | > loss_kl: 2.4114556312561035 (2.5165756422540415) + | > loss_feat: 3.955848455429077 (3.6741489864620767) + | > loss_mel: 20.802043914794922 (20.86840102606613) + | > loss_duration: 1.3896381855010986 (1.361025849114294) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.553224563598633 (30.335974037411656) + | > grad_norm_1: tensor(243.0027, device='cuda:0') (tensor(141.9591, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 1.0744 (0.9070225049384498) + | > loader_time: 0.0167 (0.014495980127055656) + + + --> TIME: 2024-06-25 08:29:10 -- STEP: 531/591 -- GLOBAL_STEP: 20625 + | > loss_disc: 2.658118486404419 (2.714429831998721) + | > loss_disc_real_0: 0.22967594861984253 (0.20875978799582426) + | > loss_disc_real_1: 0.23100067675113678 (0.2271942853534738) + | > loss_disc_real_2: 0.2382604479789734 (0.23533477284814008) + | > loss_disc_real_3: 0.22962342202663422 (0.2350697894273729) + | > loss_disc_real_4: 0.23888038098812103 (0.23309659031824878) + | > loss_disc_real_5: 0.24508249759674072 (0.23892323683996433) + | > loss_0: 2.658118486404419 (2.714429831998721) + | > grad_norm_0: tensor(8.2280, device='cuda:0') (tensor(10.1924, device='cuda:0')) + | > loss_gen: 1.8582773208618164 (1.9155136923780998) + | > loss_kl: 2.476814031600952 (2.5150295601501753) + | > loss_feat: 3.603308916091919 (3.6715018394527688) + | > loss_mel: 19.870805740356445 (20.857678634298743) + | > loss_duration: 1.4237631559371948 (1.3626045691315964) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.232967376708984 (30.322328272961403) + | > grad_norm_1: tensor(88.8945, device='cuda:0') (tensor(141.4718, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 1.0967 (0.9159955542864323) + | > loader_time: 0.0188 (0.014761559707296774) + + + --> TIME: 2024-06-25 08:29:39 -- STEP: 556/591 -- GLOBAL_STEP: 20650 + | > loss_disc: 2.7406842708587646 (2.7149244058046413) + | > loss_disc_real_0: 0.1544661819934845 (0.20862344380387732) + | > loss_disc_real_1: 0.2545013427734375 (0.22718917648140474) + | > loss_disc_real_2: 0.24856962263584137 (0.23539989532755434) + | > loss_disc_real_3: 0.2520870566368103 (0.23513046748560967) + | > loss_disc_real_4: 0.21909362077713013 (0.23320717908602825) + | > loss_disc_real_5: 0.22294844686985016 (0.23900280517639874) + | > loss_0: 2.7406842708587646 (2.7149244058046413) + | > grad_norm_0: tensor(11.7246, device='cuda:0') (tensor(10.1743, device='cuda:0')) + | > loss_gen: 2.011314868927002 (1.9146008590142505) + | > loss_kl: 2.4414381980895996 (2.512596010732994) + | > loss_feat: 3.4566280841827393 (3.662867725324287) + | > loss_mel: 20.583538055419922 (20.83718361271374) + | > loss_duration: 1.4165863990783691 (1.364297181582279) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.90950584411621 (30.29154535677793) + | > grad_norm_1: tensor(99.2246, device='cuda:0') (tensor(139.7992, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 1.1531 (0.9261945640440468) + | > loader_time: 0.0198 (0.015068155827282147) + + + --> TIME: 2024-06-25 08:30:10 -- STEP: 581/591 -- GLOBAL_STEP: 20675 + | > loss_disc: 2.767822742462158 (2.715257191206626) + | > loss_disc_real_0: 0.3368765413761139 (0.208809235073921) + | > loss_disc_real_1: 0.1747598946094513 (0.2272380610956793) + | > loss_disc_real_2: 0.1890750229358673 (0.23534169169896235) + | > loss_disc_real_3: 0.19664379954338074 (0.23511135372976016) + | > loss_disc_real_4: 0.23385308682918549 (0.23329361062657197) + | > loss_disc_real_5: 0.23315756022930145 (0.23909671586680126) + | > loss_0: 2.767822742462158 (2.715257191206626) + | > grad_norm_0: tensor(28.4659, device='cuda:0') (tensor(10.2145, device='cuda:0')) + | > loss_gen: 1.6543292999267578 (1.9136980981301523) + | > loss_kl: 2.423159122467041 (2.5105935370039822) + | > loss_feat: 3.393162965774536 (3.6537655795092427) + | > loss_mel: 20.43692398071289 (20.824897050447213) + | > loss_duration: 1.3995949029922485 (1.365960609071636) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.307170867919922 (30.268914838255785) + | > grad_norm_1: tensor(126.8624, device='cuda:0') (tensor(138.6338, device='cuda:0')) + | > current_lr_0: 0.00019915175078976256 + | > current_lr_1: 0.00019915175078976256 + | > step_time: 1.2287 (0.9377163494720722) + | > loader_time: 0.0299 (0.015432810003704132) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035465473824359 (-0.00024543417261001926) + | > avg_loss_disc: 2.7192357144457233 (-0.01254144627997178) + | > avg_loss_disc_real_0: 0.22448400740927837 (-0.04169055090305657) + | > avg_loss_disc_real_1: 0.2508364503688002 (+0.07345305224682436) + | > avg_loss_disc_real_2: 0.2576759032746579 (+0.0072234142967995685) + | > avg_loss_disc_real_3: 0.25368457557039065 (-0.015137324941919217) + | > avg_loss_disc_real_4: 0.2554805833608546 (+0.034648906043235284) + | > avg_loss_disc_real_5: 0.25408888877706315 (-0.01116143293837285) + | > avg_loss_0: 2.7192357144457233 (-0.01254144627997178) + | > avg_loss_gen: 1.9985321303631396 (+0.033584861045187875) + | > avg_loss_kl: 2.5749852403681324 (+0.02928036324521255) + | > avg_loss_feat: 3.448798864445788 (-0.2422326067660716) + | > avg_loss_mel: 21.008616711230985 (+0.008722792280472902) + | > avg_loss_duration: 1.376217258737442 (-0.00897067151171127) + | > avg_loss_1: 30.407150268554688 (-0.17961514249761024) + + + > EPOCH: 36/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 08:30:30)  + + --> TIME: 2024-06-25 08:30:43 -- STEP: 15/591 -- GLOBAL_STEP: 20700 + | > loss_disc: 2.7171294689178467 (2.709237798055013) + | > loss_disc_real_0: 0.19631999731063843 (0.2078931083281835) + | > loss_disc_real_1: 0.21574991941452026 (0.2241420030593872) + | > loss_disc_real_2: 0.2753833830356598 (0.23601862887541453) + | > loss_disc_real_3: 0.23038868606090546 (0.23460086782773334) + | > loss_disc_real_4: 0.24051092565059662 (0.2324858456850052) + | > loss_disc_real_5: 0.2355758249759674 (0.24227009415626527) + | > loss_0: 2.7171294689178467 (2.709237798055013) + | > grad_norm_0: tensor(6.9700, device='cuda:0') (tensor(8.6921, device='cuda:0')) + | > loss_gen: 1.872631549835205 (1.930170973141988) + | > loss_kl: 2.6301662921905518 (2.630383523305257) + | > loss_feat: 3.7258880138397217 (3.8735637187957765) + | > loss_mel: 20.75660514831543 (21.087888209025063) + | > loss_duration: 1.3221170902252197 (1.3310843308766682) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.30740737915039 (30.853090667724608) + | > grad_norm_1: tensor(112.7193, device='cuda:0') (tensor(117.9016, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.7772 (0.7714826265970866) + | > loader_time: 0.0093 (0.00885634422302246) + + + --> TIME: 2024-06-25 08:31:03 -- STEP: 40/591 -- GLOBAL_STEP: 20725 + | > loss_disc: 2.7202818393707275 (2.7258746802806857) + | > loss_disc_real_0: 0.21095895767211914 (0.2122445922344923) + | > loss_disc_real_1: 0.23272006213665009 (0.227173313498497) + | > loss_disc_real_2: 0.22242745757102966 (0.2350289762020111) + | > loss_disc_real_3: 0.2272798866033554 (0.23378607258200645) + | > loss_disc_real_4: 0.21832266449928284 (0.23300954550504685) + | > loss_disc_real_5: 0.21547046303749084 (0.24070291072130204) + | > loss_0: 2.7202818393707275 (2.7258746802806857) + | > grad_norm_0: tensor(4.0308, device='cuda:0') (tensor(9.7884, device='cuda:0')) + | > loss_gen: 1.845228672027588 (1.9193741738796235) + | > loss_kl: 2.6429059505462646 (2.6164012849330898) + | > loss_feat: 3.548151731491089 (3.7949464321136475) + | > loss_mel: 21.26336097717285 (21.040509366989134) + | > loss_duration: 1.3724546432495117 (1.3343455612659452) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.672100067138672 (30.705576992034914) + | > grad_norm_1: tensor(204.5099, device='cuda:0') (tensor(109.8602, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.7991 (0.7845771312713623) + | > loader_time: 0.0098 (0.009454399347305298) + + + --> TIME: 2024-06-25 08:31:24 -- STEP: 65/591 -- GLOBAL_STEP: 20750 + | > loss_disc: 2.7173049449920654 (2.7272692386920636) + | > loss_disc_real_0: 0.18068023025989532 (0.21047341995514357) + | > loss_disc_real_1: 0.2522788345813751 (0.22729103771539835) + | > loss_disc_real_2: 0.23607328534126282 (0.23743992631251995) + | > loss_disc_real_3: 0.24018535017967224 (0.23510063221821417) + | > loss_disc_real_4: 0.23508383333683014 (0.23377516728181105) + | > loss_disc_real_5: 0.22857540845870972 (0.24031883753263034) + | > loss_0: 2.7173049449920654 (2.7272692386920636) + | > grad_norm_0: tensor(6.7251, device='cuda:0') (tensor(8.8559, device='cuda:0')) + | > loss_gen: 2.0046439170837402 (1.9172234095059908) + | > loss_kl: 2.60428786277771 (2.587984822346615) + | > loss_feat: 3.767108917236328 (3.7583303048060492) + | > loss_mel: 21.31314468383789 (21.019268652108998) + | > loss_duration: 1.3677769899368286 (1.3427267276323758) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.056962966918945 (30.625533940241887) + | > grad_norm_1: tensor(49.6549, device='cuda:0') (tensor(125.7670, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.816 (0.798028249007005) + | > loader_time: 0.0104 (0.009753817778367263) + + + --> TIME: 2024-06-25 08:31:45 -- STEP: 90/591 -- GLOBAL_STEP: 20775 + | > loss_disc: 2.805126667022705 (2.7242119391759236) + | > loss_disc_real_0: 0.20883053541183472 (0.20949550974700187) + | > loss_disc_real_1: 0.20462164282798767 (0.2262472035156356) + | > loss_disc_real_2: 0.24106931686401367 (0.23742852376566992) + | > loss_disc_real_3: 0.22952929139137268 (0.23499258044693205) + | > loss_disc_real_4: 0.2669636309146881 (0.23389272607035108) + | > loss_disc_real_5: 0.28711625933647156 (0.24062342991431554) + | > loss_0: 2.805126667022705 (2.7242119391759236) + | > grad_norm_0: tensor(4.1917, device='cuda:0') (tensor(9.4459, device='cuda:0')) + | > loss_gen: 1.8260278701782227 (1.9160156581136916) + | > loss_kl: 2.5272130966186523 (2.5742423587375223) + | > loss_feat: 3.390920877456665 (3.7606603119108413) + | > loss_mel: 20.8491153717041 (20.998798667059997) + | > loss_duration: 1.371034860610962 (1.3498179462220938) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.964311599731445 (30.59953490363227) + | > grad_norm_1: tensor(253.5577, device='cuda:0') (tensor(128.5294, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.8306 (0.805391870604621) + | > loader_time: 0.0114 (0.010019789801703559) + + + --> TIME: 2024-06-25 08:32:07 -- STEP: 115/591 -- GLOBAL_STEP: 20800 + | > loss_disc: 2.7264904975891113 (2.7275832632313604) + | > loss_disc_real_0: 0.19136051833629608 (0.20965673644905505) + | > loss_disc_real_1: 0.2201351374387741 (0.22767759276473004) + | > loss_disc_real_2: 0.2504551112651825 (0.23813819185547205) + | > loss_disc_real_3: 0.26379239559173584 (0.23474896148495053) + | > loss_disc_real_4: 0.22948826849460602 (0.23333937072235605) + | > loss_disc_real_5: 0.22530214488506317 (0.24080451040164283) + | > loss_0: 2.7264904975891113 (2.7275832632313604) + | > grad_norm_0: tensor(5.5073, device='cuda:0') (tensor(10.0232, device='cuda:0')) + | > loss_gen: 1.9758388996124268 (1.9147405624389648) + | > loss_kl: 2.511021375656128 (2.554567363987799) + | > loss_feat: 3.9647879600524902 (3.7468309713446577) + | > loss_mel: 21.813018798828125 (20.989325896553364) + | > loss_duration: 1.3662892580032349 (1.3528306805569192) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.630956649780273 (30.55829547384511) + | > grad_norm_1: tensor(43.7791, device='cuda:0') (tensor(126.7654, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.8716 (0.8124488623245902) + | > loader_time: 0.0111 (0.010325085598489513) + + + --> TIME: 2024-06-25 08:32:28 -- STEP: 140/591 -- GLOBAL_STEP: 20825 + | > loss_disc: 2.678921699523926 (2.7245512945311408) + | > loss_disc_real_0: 0.14669480919837952 (0.2092334451952151) + | > loss_disc_real_1: 0.2396140843629837 (0.22778151535562108) + | > loss_disc_real_2: 0.1961652934551239 (0.23739854938217572) + | > loss_disc_real_3: 0.22776809334754944 (0.234529902360269) + | > loss_disc_real_4: 0.2405175268650055 (0.23279974226440703) + | > loss_disc_real_5: 0.22310283780097961 (0.23988377824425697) + | > loss_0: 2.678921699523926 (2.7245512945311408) + | > grad_norm_0: tensor(12.9115, device='cuda:0') (tensor(9.8410, device='cuda:0')) + | > loss_gen: 2.122830629348755 (1.9151554601533072) + | > loss_kl: 2.5326642990112305 (2.549571187155588) + | > loss_feat: 3.7819247245788574 (3.7443587967327665) + | > loss_mel: 20.574634552001953 (20.98493018831525) + | > loss_duration: 1.3319313526153564 (1.354054893766131) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.34398651123047 (30.548070512499127) + | > grad_norm_1: tensor(103.5600, device='cuda:0') (tensor(125.5184, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.8483 (0.818052635874067) + | > loader_time: 0.0113 (0.010585762773241315) + + + --> TIME: 2024-06-25 08:32:50 -- STEP: 165/591 -- GLOBAL_STEP: 20850 + | > loss_disc: 2.7524406909942627 (2.7273945519418423) + | > loss_disc_real_0: 0.2664039731025696 (0.209891152291587) + | > loss_disc_real_1: 0.1995667666196823 (0.22801425592465835) + | > loss_disc_real_2: 0.22047346830368042 (0.23764860928058623) + | > loss_disc_real_3: 0.19513684511184692 (0.23533524211609003) + | > loss_disc_real_4: 0.24037092924118042 (0.23326055958415523) + | > loss_disc_real_5: 0.2625426948070526 (0.24043928565401018) + | > loss_0: 2.7524406909942627 (2.7273945519418423) + | > grad_norm_0: tensor(12.6368, device='cuda:0') (tensor(9.9368, device='cuda:0')) + | > loss_gen: 1.7398368120193481 (1.912812792893612) + | > loss_kl: 2.475858688354492 (2.5407113783287274) + | > loss_feat: 3.4804697036743164 (3.7339189977356884) + | > loss_mel: 20.523998260498047 (20.97105181144946) + | > loss_duration: 1.327945590019226 (1.351883530616761) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.548110961914062 (30.510378577492453) + | > grad_norm_1: tensor(36.5689, device='cuda:0') (tensor(126.0828, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.8555 (0.8244476809646144) + | > loader_time: 0.0137 (0.010872098171349733) + + + --> TIME: 2024-06-25 08:33:12 -- STEP: 190/591 -- GLOBAL_STEP: 20875 + | > loss_disc: 2.6824514865875244 (2.7242967894202783) + | > loss_disc_real_0: 0.3079834580421448 (0.2096805080771447) + | > loss_disc_real_1: 0.2704828083515167 (0.22771518504933308) + | > loss_disc_real_2: 0.2348325401544571 (0.23744457567992963) + | > loss_disc_real_3: 0.2590831518173218 (0.2353698012075926) + | > loss_disc_real_4: 0.22970230877399445 (0.23295694095523733) + | > loss_disc_real_5: 0.2278459519147873 (0.2397672879068475) + | > loss_0: 2.6824514865875244 (2.7242967894202783) + | > grad_norm_0: tensor(14.0035, device='cuda:0') (tensor(9.9494, device='cuda:0')) + | > loss_gen: 1.9906883239746094 (1.9141889145499782) + | > loss_kl: 2.610380172729492 (2.535191607475281) + | > loss_feat: 3.847646951675415 (3.736989951133728) + | > loss_mel: 21.238271713256836 (20.97286396026612) + | > loss_duration: 1.3572921752929688 (1.350451794423555) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.044279098510742 (30.509686269258196) + | > grad_norm_1: tensor(242.5528, device='cuda:0') (tensor(133.1625, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.8621 (0.8299388358467503) + | > loader_time: 0.0121 (0.011107552678961508) + + + --> TIME: 2024-06-25 08:33:34 -- STEP: 215/591 -- GLOBAL_STEP: 20900 + | > loss_disc: 2.660615921020508 (2.721773782996244) + | > loss_disc_real_0: 0.2016756683588028 (0.2095882616763892) + | > loss_disc_real_1: 0.18464425206184387 (0.22738679155360822) + | > loss_disc_real_2: 0.24547992646694183 (0.23707197890725246) + | > loss_disc_real_3: 0.22028324007987976 (0.23499764210956042) + | > loss_disc_real_4: 0.21437036991119385 (0.23281395068002303) + | > loss_disc_real_5: 0.24452072381973267 (0.23991837723310602) + | > loss_0: 2.660615921020508 (2.721773782996244) + | > grad_norm_0: tensor(4.2211, device='cuda:0') (tensor(9.9126, device='cuda:0')) + | > loss_gen: 1.9367156028747559 (1.913331215880638) + | > loss_kl: 2.5764920711517334 (2.5302810602409895) + | > loss_feat: 3.8036575317382812 (3.733065124999645) + | > loss_mel: 20.657081604003906 (20.958826659446544) + | > loss_duration: 1.3841936588287354 (1.3498173076053) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.35814094543457 (30.485321408648822) + | > grad_norm_1: tensor(110.9233, device='cuda:0') (tensor(128.6435, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.8818 (0.8355475425720215) + | > loader_time: 0.0145 (0.011348310736722728) + + + --> TIME: 2024-06-25 08:33:57 -- STEP: 240/591 -- GLOBAL_STEP: 20925 + | > loss_disc: 2.709075927734375 (2.7196622282266616) + | > loss_disc_real_0: 0.17557336390018463 (0.20883941923578586) + | > loss_disc_real_1: 0.28645139932632446 (0.2274023968105515) + | > loss_disc_real_2: 0.23132763803005219 (0.23668710105121135) + | > loss_disc_real_3: 0.2395821362733841 (0.23512838886429865) + | > loss_disc_real_4: 0.23928000032901764 (0.23276003344605367) + | > loss_disc_real_5: 0.22224825620651245 (0.2398250302920739) + | > loss_0: 2.709075927734375 (2.7196622282266616) + | > grad_norm_0: tensor(5.7989, device='cuda:0') (tensor(9.8266, device='cuda:0')) + | > loss_gen: 1.968813419342041 (1.9134418820341428) + | > loss_kl: 2.551042318344116 (2.5294186919927597) + | > loss_feat: 3.934508800506592 (3.7320890466372174) + | > loss_mel: 21.02401351928711 (20.954111711184186) + | > loss_duration: 1.3653836250305176 (1.3496003846327465) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.843759536743164 (30.478661727905273) + | > grad_norm_1: tensor(168.0116, device='cuda:0') (tensor(126.3921, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.8946 (0.8411581536134084) + | > loader_time: 0.0151 (0.011681462327639265) + + + --> TIME: 2024-06-25 08:34:20 -- STEP: 265/591 -- GLOBAL_STEP: 20950 + | > loss_disc: 2.6227285861968994 (2.7208393294856235) + | > loss_disc_real_0: 0.21519845724105835 (0.20994973573482267) + | > loss_disc_real_1: 0.2058240920305252 (0.22727169186439156) + | > loss_disc_real_2: 0.18874679505825043 (0.23639702515782052) + | > loss_disc_real_3: 0.2162913978099823 (0.23495993811004567) + | > loss_disc_real_4: 0.2129998356103897 (0.23269449939142983) + | > loss_disc_real_5: 0.22390896081924438 (0.23975449198821805) + | > loss_0: 2.6227285861968994 (2.7208393294856235) + | > grad_norm_0: tensor(6.2635, device='cuda:0') (tensor(10.4520, device='cuda:0')) + | > loss_gen: 1.8131468296051025 (1.914111237256032) + | > loss_kl: 2.388640880584717 (2.5295964816831193) + | > loss_feat: 3.736659049987793 (3.7300950878071335) + | > loss_mel: 20.45038414001465 (20.94765297511839) + | > loss_duration: 1.3210294246673584 (1.3495736239091405) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.70985984802246 (30.47102942556705) + | > grad_norm_1: tensor(186.6047, device='cuda:0') (tensor(135.4844, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.9179 (0.8466954852050206) + | > loader_time: 0.0145 (0.01194625260694972) + + + --> TIME: 2024-06-25 08:34:43 -- STEP: 290/591 -- GLOBAL_STEP: 20975 + | > loss_disc: 2.711527109146118 (2.7206866519204502) + | > loss_disc_real_0: 0.2939556837081909 (0.20966342756974285) + | > loss_disc_real_1: 0.19797565042972565 (0.22737203425374525) + | > loss_disc_real_2: 0.21387477219104767 (0.23651439449910464) + | > loss_disc_real_3: 0.22931811213493347 (0.23495179404472483) + | > loss_disc_real_4: 0.25739529728889465 (0.23278180679370616) + | > loss_disc_real_5: 0.2334607094526291 (0.23967855634360477) + | > loss_0: 2.711527109146118 (2.7206866519204502) + | > grad_norm_0: tensor(18.8179, device='cuda:0') (tensor(10.2834, device='cuda:0')) + | > loss_gen: 1.860597848892212 (1.9152379833418747) + | > loss_kl: 2.6160528659820557 (2.528580406616474) + | > loss_feat: 3.7931935787200928 (3.730219807295964) + | > loss_mel: 21.546146392822266 (20.95326832080711) + | > loss_duration: 1.3034874200820923 (1.3502824002298819) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.119478225708008 (30.477588956109408) + | > grad_norm_1: tensor(57.5728, device='cuda:0') (tensor(137.0271, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.9121 (0.8520806271454383) + | > loader_time: 0.0167 (0.012216844229862614) + + + --> TIME: 2024-06-25 08:35:07 -- STEP: 315/591 -- GLOBAL_STEP: 21000 + | > loss_disc: 2.729351282119751 (2.72058873933459) + | > loss_disc_real_0: 0.118528813123703 (0.209460358108793) + | > loss_disc_real_1: 0.24322865903377533 (0.2272532379343396) + | > loss_disc_real_2: 0.25498008728027344 (0.23665096102252844) + | > loss_disc_real_3: 0.26047250628471375 (0.2353582734626437) + | > loss_disc_real_4: 0.21194635331630707 (0.23260637025038403) + | > loss_disc_real_5: 0.24391792714595795 (0.2395960687644898) + | > loss_0: 2.729351282119751 (2.72058873933459) + | > grad_norm_0: tensor(19.4584, device='cuda:0') (tensor(10.4052, device='cuda:0')) + | > loss_gen: 1.9480791091918945 (1.9151103693341451) + | > loss_kl: 2.569880247116089 (2.529694056889367) + | > loss_feat: 3.4104626178741455 (3.726012688591367) + | > loss_mel: 19.71604347229004 (20.944070985582155) + | > loss_duration: 1.3580098152160645 (1.3507831827042596) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.00247573852539 (30.46567133343409) + | > grad_norm_1: tensor(168.2081, device='cuda:0') (tensor(135.6453, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.9088 (0.8579417122734918) + | > loader_time: 0.0142 (0.012474049462212464) + + + --> TIME: 2024-06-25 08:35:31 -- STEP: 340/591 -- GLOBAL_STEP: 21025 + | > loss_disc: 2.6915712356567383 (2.719190997937145) + | > loss_disc_real_0: 0.21961098909378052 (0.20928480850861353) + | > loss_disc_real_1: 0.2263258993625641 (0.22731265140806928) + | > loss_disc_real_2: 0.2308257520198822 (0.23642926579889134) + | > loss_disc_real_3: 0.21810080111026764 (0.23511853191782447) + | > loss_disc_real_4: 0.23160947859287262 (0.2326602441861349) + | > loss_disc_real_5: 0.2611047923564911 (0.23948891281205065) + | > loss_0: 2.6915712356567383 (2.719190997937145) + | > grad_norm_0: tensor(5.1198, device='cuda:0') (tensor(10.2636, device='cuda:0')) + | > loss_gen: 1.8937370777130127 (1.914896430688746) + | > loss_kl: 2.5763468742370605 (2.525139357763177) + | > loss_feat: 3.5700151920318604 (3.7243328234728645) + | > loss_mel: 20.84279441833496 (20.940562960680808) + | > loss_duration: 1.3607141971588135 (1.351982249933131) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.243608474731445 (30.456913858301498) + | > grad_norm_1: tensor(77.0483, device='cuda:0') (tensor(133.0073, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.9794 (0.8633890635827008) + | > loader_time: 0.014 (0.01270551330902998) + + + --> TIME: 2024-06-25 08:35:55 -- STEP: 365/591 -- GLOBAL_STEP: 21050 + | > loss_disc: 2.6962506771087646 (2.718692795871053) + | > loss_disc_real_0: 0.19506466388702393 (0.20892036004017478) + | > loss_disc_real_1: 0.23817448318004608 (0.22737821699821786) + | > loss_disc_real_2: 0.2348683774471283 (0.23658770518760164) + | > loss_disc_real_3: 0.2506227195262909 (0.23521850460196195) + | > loss_disc_real_4: 0.2485039234161377 (0.2326651940607045) + | > loss_disc_real_5: 0.24748870730400085 (0.2394501557088878) + | > loss_0: 2.6962506771087646 (2.718692795871053) + | > grad_norm_0: tensor(5.7391, device='cuda:0') (tensor(10.0648, device='cuda:0')) + | > loss_gen: 1.8674260377883911 (1.9138257653745887) + | > loss_kl: 2.4861462116241455 (2.524371039377499) + | > loss_feat: 3.7175955772399902 (3.7153069750903405) + | > loss_mel: 21.463848114013672 (20.930996536881967) + | > loss_duration: 1.3634405136108398 (1.3535974091046483) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.898456573486328 (30.438097752610297) + | > grad_norm_1: tensor(217.3153, device='cuda:0') (tensor(132.4271, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.9419 (0.8696982612348583) + | > loader_time: 0.0182 (0.012955672773596365) + + + --> TIME: 2024-06-25 08:36:20 -- STEP: 390/591 -- GLOBAL_STEP: 21075 + | > loss_disc: 2.8298115730285645 (2.7187157637033685) + | > loss_disc_real_0: 0.12726061046123505 (0.20884069775541622) + | > loss_disc_real_1: 0.26696500182151794 (0.22732039869595796) + | > loss_disc_real_2: 0.23284584283828735 (0.2365294396495208) + | > loss_disc_real_3: 0.2897583544254303 (0.23518966944553912) + | > loss_disc_real_4: 0.24598392844200134 (0.23273148097288915) + | > loss_disc_real_5: 0.253657728433609 (0.23947482842665452) + | > loss_0: 2.8298115730285645 (2.7187157637033685) + | > grad_norm_0: tensor(16.7889, device='cuda:0') (tensor(10.1075, device='cuda:0')) + | > loss_gen: 2.176192283630371 (1.9148427431400006) + | > loss_kl: 2.505382537841797 (2.522648188395377) + | > loss_feat: 3.559018135070801 (3.7117722609104256) + | > loss_mel: 20.547805786132812 (20.918907820872782) + | > loss_duration: 1.390176773071289 (1.354457059273353) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.17857551574707 (30.422628114162347) + | > grad_norm_1: tensor(251.9285, device='cuda:0') (tensor(133.2043, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.9556 (0.8761950847430107) + | > loader_time: 0.018 (0.01324973106384278) + + + --> TIME: 2024-06-25 08:36:45 -- STEP: 415/591 -- GLOBAL_STEP: 21100 + | > loss_disc: 2.7261791229248047 (2.7177510721137703) + | > loss_disc_real_0: 0.14453424513339996 (0.2084936929215868) + | > loss_disc_real_1: 0.22429491579532623 (0.22735211892300342) + | > loss_disc_real_2: 0.2434891015291214 (0.2364964601864298) + | > loss_disc_real_3: 0.1966719925403595 (0.23515738625842406) + | > loss_disc_real_4: 0.2343185544013977 (0.2328195849815047) + | > loss_disc_real_5: 0.23794525861740112 (0.23938829751617938) + | > loss_0: 2.7261791229248047 (2.7177510721137703) + | > grad_norm_0: tensor(19.2512, device='cuda:0') (tensor(10.1262, device='cuda:0')) + | > loss_gen: 2.1239514350891113 (1.9144493255270534) + | > loss_kl: 2.4102606773376465 (2.5205534412200175) + | > loss_feat: 3.65336012840271 (3.7052055835723876) + | > loss_mel: 21.044313430786133 (20.90844091392426) + | > loss_duration: 1.384740948677063 (1.3554014998746207) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.61662483215332 (30.40405079485422) + | > grad_norm_1: tensor(57.6603, device='cuda:0') (tensor(132.6559, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 1.0472 (0.8826179349278829) + | > loader_time: 0.0189 (0.013519160718802952) + + + --> TIME: 2024-06-25 08:37:11 -- STEP: 440/591 -- GLOBAL_STEP: 21125 + | > loss_disc: 2.826730728149414 (2.7178107662634394) + | > loss_disc_real_0: 0.25853925943374634 (0.20873346863822503) + | > loss_disc_real_1: 0.22156071662902832 (0.22741869125853886) + | > loss_disc_real_2: 0.21746617555618286 (0.23633407222276387) + | > loss_disc_real_3: 0.29924339056015015 (0.23532279794189062) + | > loss_disc_real_4: 0.28502151370048523 (0.23289503753185273) + | > loss_disc_real_5: 0.29562780261039734 (0.23939885232936253) + | > loss_0: 2.826730728149414 (2.7178107662634394) + | > grad_norm_0: tensor(8.0366, device='cuda:0') (tensor(10.2567, device='cuda:0')) + | > loss_gen: 1.8838527202606201 (1.9147390717809851) + | > loss_kl: 2.4064595699310303 (2.5185656580057993) + | > loss_feat: 3.0902926921844482 (3.703747662089088) + | > loss_mel: 20.7366943359375 (20.90886520905929) + | > loss_duration: 1.3593791723251343 (1.3565283241597088) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.47667694091797 (30.402445949207653) + | > grad_norm_1: tensor(128.7121, device='cuda:0') (tensor(131.8052, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 1.0398 (0.8897134201093153) + | > loader_time: 0.0199 (0.013795589316974994) + + + --> TIME: 2024-06-25 08:37:37 -- STEP: 465/591 -- GLOBAL_STEP: 21150 + | > loss_disc: 2.7287254333496094 (2.7176527289934036) + | > loss_disc_real_0: 0.16271980106830597 (0.20853593384706845) + | > loss_disc_real_1: 0.24641236662864685 (0.22751870841108343) + | > loss_disc_real_2: 0.23347248136997223 (0.236186027622992) + | > loss_disc_real_3: 0.27161794900894165 (0.23538060652953322) + | > loss_disc_real_4: 0.2794814705848694 (0.23283874171395455) + | > loss_disc_real_5: 0.24729204177856445 (0.23931518600833032) + | > loss_0: 2.7287254333496094 (2.7176527289934036) + | > grad_norm_0: tensor(10.9582, device='cuda:0') (tensor(10.2308, device='cuda:0')) + | > loss_gen: 1.9383032321929932 (1.914159342037734) + | > loss_kl: 2.4818639755249023 (2.51696271383634) + | > loss_feat: 3.551758050918579 (3.6976095297003306) + | > loss_mel: 20.30402946472168 (20.891375371461283) + | > loss_duration: 1.3751299381256104 (1.3575572926511046) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.65108299255371 (30.377664250199512) + | > grad_norm_1: tensor(83.1268, device='cuda:0') (tensor(131.5019, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 1.0293 (0.8970224062601725) + | > loader_time: 0.0192 (0.014090334984564018) + + + --> TIME: 2024-06-25 08:38:04 -- STEP: 490/591 -- GLOBAL_STEP: 21175 + | > loss_disc: 2.739628314971924 (2.7171290076508794) + | > loss_disc_real_0: 0.35387375950813293 (0.20862259013312204) + | > loss_disc_real_1: 0.25776153802871704 (0.22734091397450895) + | > loss_disc_real_2: 0.21147139370441437 (0.23618286835904026) + | > loss_disc_real_3: 0.2537643015384674 (0.23526743908925932) + | > loss_disc_real_4: 0.2688084542751312 (0.23292584857162163) + | > loss_disc_real_5: 0.2610754370689392 (0.23932673131324808) + | > loss_0: 2.739628314971924 (2.7171290076508794) + | > grad_norm_0: tensor(22.7897, device='cuda:0') (tensor(10.3334, device='cuda:0')) + | > loss_gen: 1.8441729545593262 (1.9138730114820053) + | > loss_kl: 2.635796546936035 (2.5163615367850465) + | > loss_feat: 3.4806814193725586 (3.692659978477322) + | > loss_mel: 20.2708740234375 (20.880286131099787) + | > loss_duration: 1.3713502883911133 (1.3588811280776043) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.602874755859375 (30.362061788597885) + | > grad_norm_1: tensor(196.0475, device='cuda:0') (tensor(132.1940, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 1.0383 (0.9045705376839152) + | > loader_time: 0.0221 (0.01439117947403266) + + + --> TIME: 2024-06-25 08:38:32 -- STEP: 515/591 -- GLOBAL_STEP: 21200 + | > loss_disc: 2.698798179626465 (2.7171932938029446) + | > loss_disc_real_0: 0.3149084150791168 (0.20855338247945007) + | > loss_disc_real_1: 0.23641835153102875 (0.2274280792590484) + | > loss_disc_real_2: 0.21901680529117584 (0.2361229551359288) + | > loss_disc_real_3: 0.23000428080558777 (0.23530670447835644) + | > loss_disc_real_4: 0.20253536105155945 (0.23290512162504845) + | > loss_disc_real_5: 0.22084079682826996 (0.23924960922268987) + | > loss_0: 2.698798179626465 (2.7171932938029446) + | > grad_norm_0: tensor(17.0966, device='cuda:0') (tensor(10.3407, device='cuda:0')) + | > loss_gen: 1.8006131649017334 (1.9130740779117472) + | > loss_kl: 2.369014024734497 (2.514461985375116) + | > loss_feat: 3.479287624359131 (3.6873504606265466) + | > loss_mel: 20.529972076416016 (20.864450958400102) + | > loss_duration: 1.40342378616333 (1.3605257874553642) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.58230972290039 (30.339863275324255) + | > grad_norm_1: tensor(65.8764, device='cuda:0') (tensor(130.9810, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 1.1088 (0.9142907823173745) + | > loader_time: 0.0227 (0.014696339264656738) + + + --> TIME: 2024-06-25 08:39:02 -- STEP: 540/591 -- GLOBAL_STEP: 21225 + | > loss_disc: 2.7424116134643555 (2.717703718167762) + | > loss_disc_real_0: 0.21608607470989227 (0.2083771002927312) + | > loss_disc_real_1: 0.25060632824897766 (0.2275466062956386) + | > loss_disc_real_2: 0.23751996457576752 (0.23620453998446464) + | > loss_disc_real_3: 0.26107847690582275 (0.2353193698105988) + | > loss_disc_real_4: 0.2368209809064865 (0.23298507214144423) + | > loss_disc_real_5: 0.22923190891742706 (0.23925182295066338) + | > loss_0: 2.7424116134643555 (2.717703718167762) + | > grad_norm_0: tensor(5.1684, device='cuda:0') (tensor(10.2680, device='cuda:0')) + | > loss_gen: 1.8317673206329346 (1.9118311314671124) + | > loss_kl: 2.478447675704956 (2.511153583173397) + | > loss_feat: 3.2531960010528564 (3.677426353648857) + | > loss_mel: 19.732778549194336 (20.848328254840997) + | > loss_duration: 1.3763868808746338 (1.3618974581912713) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.672576904296875 (30.310636795891657) + | > grad_norm_1: tensor(191.8474, device='cuda:0') (tensor(130.4081, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 1.2421 (0.9259227183130053) + | > loader_time: 0.0203 (0.015022264145038752) + + + --> TIME: 2024-06-25 08:39:33 -- STEP: 565/591 -- GLOBAL_STEP: 21250 + | > loss_disc: 2.722815990447998 (2.7181653832967276) + | > loss_disc_real_0: 0.2952415347099304 (0.20849630736407984) + | > loss_disc_real_1: 0.2296055108308792 (0.22764602983947338) + | > loss_disc_real_2: 0.22466832399368286 (0.2361923626566355) + | > loss_disc_real_3: 0.25811558961868286 (0.235416269513358) + | > loss_disc_real_4: 0.24454492330551147 (0.2330929319151735) + | > loss_disc_real_5: 0.24830061197280884 (0.2393253975473674) + | > loss_0: 2.722815990447998 (2.7181653832967276) + | > grad_norm_0: tensor(13.0338, device='cuda:0') (tensor(10.3617, device='cuda:0')) + | > loss_gen: 1.7998316287994385 (1.9111058585411675) + | > loss_kl: 2.4154183864593506 (2.508560938961737) + | > loss_feat: 3.514106273651123 (3.6689557894141274) + | > loss_mel: 20.707347869873047 (20.830235999453386) + | > loss_duration: 1.4227616786956787 (1.363575502834489) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.859464645385742 (30.282434088782928) + | > grad_norm_1: tensor(341.2636, device='cuda:0') (tensor(130.8933, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 1.2291 (0.9388847570503708) + | > loader_time: 0.0254 (0.015386083484750939) + + + --> TIME: 2024-06-25 08:40:05 -- STEP: 590/591 -- GLOBAL_STEP: 21275 + | > loss_disc: 2.715588331222534 (2.717864820108573) + | > loss_disc_real_0: 0.2039296180009842 (0.20840514100204086) + | > loss_disc_real_1: 0.2384575456380844 (0.22753651990728854) + | > loss_disc_real_2: 0.269120454788208 (0.23609151062318834) + | > loss_disc_real_3: 0.30167797207832336 (0.23541252082687308) + | > loss_disc_real_4: 0.2335096001625061 (0.23303516388444578) + | > loss_disc_real_5: 0.27157333493232727 (0.23935082915980938) + | > loss_0: 2.715588331222534 (2.717864820108573) + | > grad_norm_0: tensor(5.9792, device='cuda:0') (tensor(10.4084, device='cuda:0')) + | > loss_gen: 1.9531611204147339 (1.9111389018721496) + | > loss_kl: 2.3393354415893555 (2.506258979086148) + | > loss_feat: 3.382711887359619 (3.663001936168994) + | > loss_mel: 20.11467170715332 (20.808181946964595) + | > loss_duration: 1.3768390417099 (1.3650951781515346) + | > amp_scaler: 512.0 (261.20677966101675) + | > loss_1: 29.166719436645508 (30.2536769511336) + | > grad_norm_1: tensor(126.7982, device='cuda:0') (tensor(131.6960, device='cuda:0')) + | > current_lr_0: 0.00019912685682091382 + | > current_lr_1: 0.00019912685682091382 + | > step_time: 0.4821 (0.9515754659297102) + | > loader_time: 0.008 (0.015717754525653396) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003638044316717919 (+9.149693428201888e-05) + | > avg_loss_disc: 2.667982360149952 (-0.05125335429577138) + | > avg_loss_disc_real_0: 0.24589165284278544 (+0.021407645433507067) + | > avg_loss_disc_real_1: 0.2338564135926835 (-0.016980036776116703) + | > avg_loss_disc_real_2: 0.2046234648912511 (-0.053052438383406786) + | > avg_loss_disc_real_3: 0.2107244045176405 (-0.04296017105275016) + | > avg_loss_disc_real_4: 0.22179764192155066 (-0.03368294143930395) + | > avg_loss_disc_real_5: 0.2336237887118725 (-0.020465100065190645) + | > avg_loss_0: 2.667982360149952 (-0.05125335429577138) + | > avg_loss_gen: 1.9084989796293543 (-0.09003315073378526) + | > avg_loss_kl: 2.6001051740443457 (+0.025119933676213346) + | > avg_loss_feat: 3.6459163655626012 (+0.19711750111681337) + | > avg_loss_mel: 20.829391520074072 (-0.17922519115691316) + | > avg_loss_duration: 1.3893697819811235 (+0.013152523243681546) + | > avg_loss_1: 30.373282006446352 (-0.033868262108335045) + + + > EPOCH: 37/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 08:40:15)  + + --> TIME: 2024-06-25 08:40:35 -- STEP: 24/591 -- GLOBAL_STEP: 21300 + | > loss_disc: 2.788717269897461 (2.7029509047667184) + | > loss_disc_real_0: 0.2362373322248459 (0.20193094139297804) + | > loss_disc_real_1: 0.2465815544128418 (0.22686480854948363) + | > loss_disc_real_2: 0.24874235689640045 (0.23717596381902695) + | > loss_disc_real_3: 0.2882671356201172 (0.2407092141608397) + | > loss_disc_real_4: 0.2556540071964264 (0.23496277071535587) + | > loss_disc_real_5: 0.2769782841205597 (0.24084657492736974) + | > loss_0: 2.788717269897461 (2.7029509047667184) + | > grad_norm_0: tensor(5.5062, device='cuda:0') (tensor(8.1364, device='cuda:0')) + | > loss_gen: 1.8309738636016846 (1.9669234951337178) + | > loss_kl: 2.7468390464782715 (2.6764518320560455) + | > loss_feat: 3.5649664402008057 (3.9519943594932556) + | > loss_mel: 20.333337783813477 (20.761796712875363) + | > loss_duration: 1.3205406665802002 (1.3300173431634907) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.796659469604492 (30.68718385696411) + | > grad_norm_1: tensor(49.0570, device='cuda:0') (tensor(135.0879, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.7814 (0.774620125691096) + | > loader_time: 0.009 (0.00875799854596456) + + + --> TIME: 2024-06-25 08:40:56 -- STEP: 49/591 -- GLOBAL_STEP: 21325 + | > loss_disc: 2.6866278648376465 (2.709446264772999) + | > loss_disc_real_0: 0.22250857949256897 (0.20539314634337716) + | > loss_disc_real_1: 0.22010917961597443 (0.22735430604341078) + | > loss_disc_real_2: 0.21556125581264496 (0.23668076736586435) + | > loss_disc_real_3: 0.23400269448757172 (0.2361418124364347) + | > loss_disc_real_4: 0.19976407289505005 (0.2322980971360693) + | > loss_disc_real_5: 0.2099645882844925 (0.23764212003776006) + | > loss_0: 2.6866278648376465 (2.709446264772999) + | > grad_norm_0: tensor(4.9804, device='cuda:0') (tensor(9.2441, device='cuda:0')) + | > loss_gen: 1.9656819105148315 (1.9371491962549638) + | > loss_kl: 2.592043876647949 (2.624542080626196) + | > loss_feat: 4.085566520690918 (3.8536930424826488) + | > loss_mel: 20.901832580566406 (20.769017550410055) + | > loss_duration: 1.3619914054870605 (1.3387242652931992) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.907115936279297 (30.523126290768992) + | > grad_norm_1: tensor(43.8478, device='cuda:0') (tensor(119.4495, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.7992 (0.7860930695825693) + | > loader_time: 0.0097 (0.009318273894640863) + + + --> TIME: 2024-06-25 08:41:16 -- STEP: 74/591 -- GLOBAL_STEP: 21350 + | > loss_disc: 2.734368324279785 (2.7078251290965722) + | > loss_disc_real_0: 0.17446810007095337 (0.20517486343915398) + | > loss_disc_real_1: 0.2448006272315979 (0.22665620615353455) + | > loss_disc_real_2: 0.21331657469272614 (0.23696961958666105) + | > loss_disc_real_3: 0.21625126898288727 (0.23511241799270785) + | > loss_disc_real_4: 0.2031582146883011 (0.2327222582456228) + | > loss_disc_real_5: 0.2417953610420227 (0.23827011077790647) + | > loss_0: 2.734368324279785 (2.7078251290965722) + | > grad_norm_0: tensor(8.5811, device='cuda:0') (tensor(10.0305, device='cuda:0')) + | > loss_gen: 2.1345534324645996 (1.934353451471071) + | > loss_kl: 2.6879591941833496 (2.594801255174585) + | > loss_feat: 3.877673625946045 (3.8355913419981262) + | > loss_mel: 21.360246658325195 (20.775654122636126) + | > loss_duration: 1.3707917928695679 (1.343334310763591) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.43122673034668 (30.483734465934134) + | > grad_norm_1: tensor(53.4264, device='cuda:0') (tensor(113.6194, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8063 (0.7963393829964304) + | > loader_time: 0.0114 (0.009732684573611696) + + + --> TIME: 2024-06-25 08:41:38 -- STEP: 99/591 -- GLOBAL_STEP: 21375 + | > loss_disc: 2.6848721504211426 (2.706661002804535) + | > loss_disc_real_0: 0.22449958324432373 (0.2064484761218832) + | > loss_disc_real_1: 0.1937088817358017 (0.2264473992164689) + | > loss_disc_real_2: 0.23407693207263947 (0.23608968113407944) + | > loss_disc_real_3: 0.23970770835876465 (0.23516044457151433) + | > loss_disc_real_4: 0.22120267152786255 (0.2325713423767475) + | > loss_disc_real_5: 0.24915163218975067 (0.2386144736800531) + | > loss_0: 2.6848721504211426 (2.706661002804535) + | > grad_norm_0: tensor(9.2201, device='cuda:0') (tensor(10.2677, device='cuda:0')) + | > loss_gen: 1.8103961944580078 (1.9292403170556733) + | > loss_kl: 2.5172336101531982 (2.5854006198921575) + | > loss_feat: 3.837409496307373 (3.8296155255250257) + | > loss_mel: 21.121273040771484 (20.804864921955144) + | > loss_duration: 1.3453969955444336 (1.3506895664966467) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.631710052490234 (30.49981092202543) + | > grad_norm_1: tensor(225.9373, device='cuda:0') (tensor(127.4911, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8268 (0.8054805187263874) + | > loader_time: 0.012 (0.01005661367165922) + + + --> TIME: 2024-06-25 08:41:59 -- STEP: 124/591 -- GLOBAL_STEP: 21400 + | > loss_disc: 2.5781819820404053 (2.705943159518703) + | > loss_disc_real_0: 0.16922785341739655 (0.20694808349494012) + | > loss_disc_real_1: 0.21862083673477173 (0.22686486871492478) + | > loss_disc_real_2: 0.21558129787445068 (0.23502828216841143) + | > loss_disc_real_3: 0.2361958771944046 (0.23533378661640228) + | > loss_disc_real_4: 0.23433704674243927 (0.2323156413772414) + | > loss_disc_real_5: 0.24909737706184387 (0.238008847880748) + | > loss_0: 2.5781819820404053 (2.705943159518703) + | > grad_norm_0: tensor(5.1564, device='cuda:0') (tensor(10.4665, device='cuda:0')) + | > loss_gen: 2.101522922515869 (1.9320204613670227) + | > loss_kl: 2.625610828399658 (2.569957567799475) + | > loss_feat: 4.155421733856201 (3.8317519464800434) + | > loss_mel: 21.321197509765625 (20.852898674626502) + | > loss_duration: 1.3687361478805542 (1.351720783979662) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.57248878479004 (30.53834939772083) + | > grad_norm_1: tensor(46.1277, device='cuda:0') (tensor(139.7938, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8367 (0.8125619965214883) + | > loader_time: 0.0116 (0.01031681030027328) + + + --> TIME: 2024-06-25 08:42:20 -- STEP: 149/591 -- GLOBAL_STEP: 21425 + | > loss_disc: 2.6824228763580322 (2.7023659712516226) + | > loss_disc_real_0: 0.13733641803264618 (0.20642309710883455) + | > loss_disc_real_1: 0.23140382766723633 (0.226479842158772) + | > loss_disc_real_2: 0.2682547867298126 (0.2348912189270826) + | > loss_disc_real_3: 0.20484882593154907 (0.23496160861229737) + | > loss_disc_real_4: 0.20616106688976288 (0.23205157484384192) + | > loss_disc_real_5: 0.22250813245773315 (0.23806444160490228) + | > loss_0: 2.6824228763580322 (2.7023659712516226) + | > grad_norm_0: tensor(13.1667, device='cuda:0') (tensor(10.3701, device='cuda:0')) + | > loss_gen: 1.957078456878662 (1.93355872567068) + | > loss_kl: 2.608957052230835 (2.5660656090550766) + | > loss_feat: 3.9773318767547607 (3.836999422751817) + | > loss_mel: 20.942520141601562 (20.86370384933164) + | > loss_duration: 1.3553036451339722 (1.3513839788884925) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.841190338134766 (30.551711511291913) + | > grad_norm_1: tensor(101.6752, device='cuda:0') (tensor(143.3776, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8487 (0.8179667172015914) + | > loader_time: 0.0115 (0.010514955392619907) + + + --> TIME: 2024-06-25 08:42:42 -- STEP: 174/591 -- GLOBAL_STEP: 21450 + | > loss_disc: 2.7171335220336914 (2.70306673817251) + | > loss_disc_real_0: 0.31549525260925293 (0.20682009339503862) + | > loss_disc_real_1: 0.22798936069011688 (0.22641103200871368) + | > loss_disc_real_2: 0.22200602293014526 (0.23484763126263672) + | > loss_disc_real_3: 0.246665820479393 (0.2351496301848313) + | > loss_disc_real_4: 0.21844056248664856 (0.23192596101555332) + | > loss_disc_real_5: 0.20926862955093384 (0.2382759409046721) + | > loss_0: 2.7171335220336914 (2.70306673817251) + | > grad_norm_0: tensor(21.0733, device='cuda:0') (tensor(10.8485, device='cuda:0')) + | > loss_gen: 1.811165452003479 (1.9330310999662026) + | > loss_kl: 2.5312390327453613 (2.559602921036467) + | > loss_feat: 3.942035675048828 (3.831858554105649) + | > loss_mel: 21.14826202392578 (20.855396489987417) + | > loss_duration: 1.3521873950958252 (1.3490652579000624) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.784889221191406 (30.52895424283784) + | > grad_norm_1: tensor(165.3943, device='cuda:0') (tensor(145.0365, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8839 (0.8241348239197132) + | > loader_time: 0.0115 (0.010729693818366389) + + + --> TIME: 2024-06-25 08:43:05 -- STEP: 199/591 -- GLOBAL_STEP: 21475 + | > loss_disc: 2.748483896255493 (2.701310380619375) + | > loss_disc_real_0: 0.17580965161323547 (0.20559317581168374) + | > loss_disc_real_1: 0.24480901658535004 (0.22636495612973545) + | > loss_disc_real_2: 0.2419092059135437 (0.2347896324479999) + | > loss_disc_real_3: 0.25044986605644226 (0.2352795040038363) + | > loss_disc_real_4: 0.25838881731033325 (0.23197259098740677) + | > loss_disc_real_5: 0.2300485074520111 (0.23803068937069208) + | > loss_0: 2.748483896255493 (2.701310380619375) + | > grad_norm_0: tensor(9.1223, device='cuda:0') (tensor(10.5214, device='cuda:0')) + | > loss_gen: 2.054260492324829 (1.9334046894581474) + | > loss_kl: 2.4701552391052246 (2.557793221881041) + | > loss_feat: 3.6434152126312256 (3.8321630080141613) + | > loss_mel: 20.123050689697266 (20.838572459005228) + | > loss_duration: 1.351269006729126 (1.348659881395311) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.64215087890625 (30.510593213028645) + | > grad_norm_1: tensor(71.1468, device='cuda:0') (tensor(136.3929, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8563 (0.8294486328585067) + | > loader_time: 0.0125 (0.010959794173887625) + + + --> TIME: 2024-06-25 08:43:27 -- STEP: 224/591 -- GLOBAL_STEP: 21500 + | > loss_disc: 2.707545042037964 (2.7005738094449043) + | > loss_disc_real_0: 0.17704397439956665 (0.20603664961111337) + | > loss_disc_real_1: 0.25773870944976807 (0.22620048121150052) + | > loss_disc_real_2: 0.22080273926258087 (0.23456696766827786) + | > loss_disc_real_3: 0.19209271669387817 (0.23492714283721788) + | > loss_disc_real_4: 0.19934692978858948 (0.23174101959115692) + | > loss_disc_real_5: 0.22630088031291962 (0.23809163877740502) + | > loss_0: 2.707545042037964 (2.7005738094449043) + | > grad_norm_0: tensor(9.7915, device='cuda:0') (tensor(10.6860, device='cuda:0')) + | > loss_gen: 1.9210466146469116 (1.933142725378275) + | > loss_kl: 2.5287058353424072 (2.555714599788188) + | > loss_feat: 3.921656370162964 (3.837845222226211) + | > loss_mel: 20.8236026763916 (20.851386027676714) + | > loss_duration: 1.35811448097229 (1.3476868788046494) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.553125381469727 (30.525775407041824) + | > grad_norm_1: tensor(161.6302, device='cuda:0') (tensor(134.1229, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8786 (0.8342789826648579) + | > loader_time: 0.0145 (0.011285857430526186) + + + --> TIME: 2024-06-25 08:43:50 -- STEP: 249/591 -- GLOBAL_STEP: 21525 + | > loss_disc: 2.7843055725097656 (2.701318805955021) + | > loss_disc_real_0: 0.2544252574443817 (0.20572548754603032) + | > loss_disc_real_1: 0.17759761214256287 (0.22667570950755153) + | > loss_disc_real_2: 0.22319260239601135 (0.23476535165645032) + | > loss_disc_real_3: 0.2176831066608429 (0.23511006560430947) + | > loss_disc_real_4: 0.23244117200374603 (0.23185827741661225) + | > loss_disc_real_5: 0.247547909617424 (0.238133252504839) + | > loss_0: 2.7843055725097656 (2.701318805955021) + | > grad_norm_0: tensor(13.0099, device='cuda:0') (tensor(10.5141, device='cuda:0')) + | > loss_gen: 1.850311279296875 (1.933573101419043) + | > loss_kl: 2.56851863861084 (2.554788907368977) + | > loss_feat: 3.5091042518615723 (3.836268383815107) + | > loss_mel: 20.608234405517578 (20.854146382894857) + | > loss_duration: 1.3822377920150757 (1.3482909997304278) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.918407440185547 (30.527067758950842) + | > grad_norm_1: tensor(55.4363, device='cuda:0') (tensor(135.6433, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8936 (0.8401778299645729) + | > loader_time: 0.0153 (0.011545412990462826) + + + --> TIME: 2024-06-25 08:44:13 -- STEP: 274/591 -- GLOBAL_STEP: 21550 + | > loss_disc: 2.7128398418426514 (2.7020156279097507) + | > loss_disc_real_0: 0.2100905478000641 (0.20570975718815834) + | > loss_disc_real_1: 0.2387152910232544 (0.22685644443887865) + | > loss_disc_real_2: 0.24011433124542236 (0.2348376340674658) + | > loss_disc_real_3: 0.24162812530994415 (0.23527468425514053) + | > loss_disc_real_4: 0.2394786775112152 (0.23203255328601297) + | > loss_disc_real_5: 0.25725090503692627 (0.2381405598708313) + | > loss_0: 2.7128398418426514 (2.7020156279097507) + | > grad_norm_0: tensor(5.1825, device='cuda:0') (tensor(10.5412, device='cuda:0')) + | > loss_gen: 1.9144911766052246 (1.932720559791927) + | > loss_kl: 2.544118881225586 (2.550899225429896) + | > loss_feat: 3.552454948425293 (3.8268553018569946) + | > loss_mel: 20.780590057373047 (20.85608643858972) + | > loss_duration: 1.3707958459854126 (1.3487037920603784) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.162450790405273 (30.51526528379343) + | > grad_norm_1: tensor(149.1623, device='cuda:0') (tensor(137.5900, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.8901 (0.8455611441257228) + | > loader_time: 0.0132 (0.011852735150469479) + + + --> TIME: 2024-06-25 08:44:36 -- STEP: 299/591 -- GLOBAL_STEP: 21575 + | > loss_disc: 2.6316678524017334 (2.700528908732743) + | > loss_disc_real_0: 0.16764169931411743 (0.2051005219187226) + | > loss_disc_real_1: 0.243573397397995 (0.2267571476390928) + | > loss_disc_real_2: 0.21037794649600983 (0.23491943296182116) + | > loss_disc_real_3: 0.23463906347751617 (0.2352090312485711) + | > loss_disc_real_4: 0.24964767694473267 (0.23193021009000248) + | > loss_disc_real_5: 0.23827724158763885 (0.23804194407917584) + | > loss_0: 2.6316678524017334 (2.700528908732743) + | > grad_norm_0: tensor(6.2511, device='cuda:0') (tensor(10.3797, device='cuda:0')) + | > loss_gen: 2.0407137870788574 (1.93285327292605) + | > loss_kl: 2.5416605472564697 (2.548029618917101) + | > loss_feat: 3.9318082332611084 (3.821739363431133) + | > loss_mel: 20.3320369720459 (20.861995550302357) + | > loss_duration: 1.3510161638259888 (1.349188955332523) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.197237014770508 (30.51380674177189) + | > grad_norm_1: tensor(190.9294, device='cuda:0') (tensor(135.7223, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.929 (0.8512742846307151) + | > loader_time: 0.0142 (0.01211291013354043) + + + --> TIME: 2024-06-25 08:44:59 -- STEP: 324/591 -- GLOBAL_STEP: 21600 + | > loss_disc: 2.7015295028686523 (2.700395598823643) + | > loss_disc_real_0: 0.2160794734954834 (0.20524183607855698) + | > loss_disc_real_1: 0.2462802231311798 (0.22667639458804956) + | > loss_disc_real_2: 0.24363689124584198 (0.23532399158050984) + | > loss_disc_real_3: 0.2398647516965866 (0.23522901686805267) + | > loss_disc_real_4: 0.24714523553848267 (0.2318752489892053) + | > loss_disc_real_5: 0.2618728280067444 (0.23777718152160998) + | > loss_0: 2.7015295028686523 (2.700395598823643) + | > grad_norm_0: tensor(5.2874, device='cuda:0') (tensor(10.4531, device='cuda:0')) + | > loss_gen: 1.940643310546875 (1.9335573529019767) + | > loss_kl: 2.5852067470550537 (2.5444107460386953) + | > loss_feat: 3.7358148097991943 (3.8199136117358266) + | > loss_mel: 20.90143394470215 (20.870821587833362) + | > loss_duration: 1.3541065454483032 (1.3499818602461868) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.5172061920166 (30.518685158388113) + | > grad_norm_1: tensor(57.6042, device='cuda:0') (tensor(137.3645, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.9232 (0.856238275398443) + | > loader_time: 0.0165 (0.012391801969504651) + + + --> TIME: 2024-06-25 08:45:23 -- STEP: 349/591 -- GLOBAL_STEP: 21625 + | > loss_disc: 2.712801694869995 (2.701489130883642) + | > loss_disc_real_0: 0.20328998565673828 (0.20527239066994293) + | > loss_disc_real_1: 0.22627311944961548 (0.22667401047012525) + | > loss_disc_real_2: 0.23879973590373993 (0.23556165896719028) + | > loss_disc_real_3: 0.28087490797042847 (0.23532102728812265) + | > loss_disc_real_4: 0.2281053364276886 (0.23193589489063765) + | > loss_disc_real_5: 0.23609545826911926 (0.23800928519243497) + | > loss_0: 2.712801694869995 (2.701489130883642) + | > grad_norm_0: tensor(3.1634, device='cuda:0') (tensor(10.5073, device='cuda:0')) + | > loss_gen: 1.8013277053833008 (1.9319729675194595) + | > loss_kl: 2.527216672897339 (2.5421883731995063) + | > loss_feat: 3.9086294174194336 (3.8108624202815715) + | > loss_mel: 21.228504180908203 (20.867682776683388) + | > loss_duration: 1.3578672409057617 (1.3513037396706964) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.82354736328125 (30.50401029614118) + | > grad_norm_1: tensor(283.4844, device='cuda:0') (tensor(134.8282, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.9341 (0.86180775281693) + | > loader_time: 0.017 (0.01266856453138641) + + + --> TIME: 2024-06-25 08:45:47 -- STEP: 374/591 -- GLOBAL_STEP: 21650 + | > loss_disc: 2.7257819175720215 (2.7021864824754056) + | > loss_disc_real_0: 0.3549902141094208 (0.20540150438241142) + | > loss_disc_real_1: 0.20412714779376984 (0.22662418491062633) + | > loss_disc_real_2: 0.252777636051178 (0.2357326016069096) + | > loss_disc_real_3: 0.18042945861816406 (0.23531677147602653) + | > loss_disc_real_4: 0.2309420108795166 (0.2320629732414363) + | > loss_disc_real_5: 0.24744613468647003 (0.23814591566190363) + | > loss_0: 2.7257819175720215 (2.7021864824754056) + | > grad_norm_0: tensor(30.6013, device='cuda:0') (tensor(10.5458, device='cuda:0')) + | > loss_gen: 1.7321313619613647 (1.930684684750868) + | > loss_kl: 2.5138354301452637 (2.538134469705469) + | > loss_feat: 3.7856557369232178 (3.8039413633193564) + | > loss_mel: 20.440555572509766 (20.860248825766828) + | > loss_duration: 1.3502217531204224 (1.35287125639737) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.82240104675293 (30.485880617151924) + | > grad_norm_1: tensor(195.0175, device='cuda:0') (tensor(134.5422, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 1.0015 (0.8675252575287846) + | > loader_time: 0.0159 (0.01297519997479444) + + + --> TIME: 2024-06-25 08:46:12 -- STEP: 399/591 -- GLOBAL_STEP: 21675 + | > loss_disc: 2.718761920928955 (2.7035063424504813) + | > loss_disc_real_0: 0.2065819501876831 (0.20531054696343595) + | > loss_disc_real_1: 0.26890549063682556 (0.22681466585263274) + | > loss_disc_real_2: 0.24325966835021973 (0.235955119879921) + | > loss_disc_real_3: 0.23916956782341003 (0.23524057469272375) + | > loss_disc_real_4: 0.2227812111377716 (0.23217228749640903) + | > loss_disc_real_5: 0.24696798622608185 (0.23826420232466886) + | > loss_0: 2.718761920928955 (2.7035063424504813) + | > grad_norm_0: tensor(6.5455, device='cuda:0') (tensor(10.5871, device='cuda:0')) + | > loss_gen: 1.8936963081359863 (1.9293201373035747) + | > loss_kl: 2.585167407989502 (2.535669836484102) + | > loss_feat: 3.2949440479278564 (3.7946699024143076) + | > loss_mel: 20.20843505859375 (20.8484084624097) + | > loss_duration: 1.3718253374099731 (1.3537846630378474) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.354068756103516 (30.461853032124072) + | > grad_norm_1: tensor(97.4629, device='cuda:0') (tensor(134.1061, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.9945 (0.8745110130549076) + | > loader_time: 0.0154 (0.013222858123014445) + + + --> TIME: 2024-06-25 08:46:38 -- STEP: 424/591 -- GLOBAL_STEP: 21700 + | > loss_disc: 2.6791234016418457 (2.7031180223204077) + | > loss_disc_real_0: 0.20703810453414917 (0.2053449764355736) + | > loss_disc_real_1: 0.22405579686164856 (0.22668275563924942) + | > loss_disc_real_2: 0.2233457863330841 (0.2357782090680217) + | > loss_disc_real_3: 0.23824486136436462 (0.235129187199867) + | > loss_disc_real_4: 0.21537595987319946 (0.23212744601352034) + | > loss_disc_real_5: 0.21046984195709229 (0.23823986501204517) + | > loss_0: 2.6791234016418457 (2.7031180223204077) + | > grad_norm_0: tensor(4.3777, device='cuda:0') (tensor(10.5845, device='cuda:0')) + | > loss_gen: 1.940265417098999 (1.9287383983720023) + | > loss_kl: 2.5644404888153076 (2.534829795922874) + | > loss_feat: 3.8751509189605713 (3.788282663192389) + | > loss_mel: 20.724119186401367 (20.834627142492344) + | > loss_duration: 1.3911316394805908 (1.3547067723746569) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.49510955810547 (30.441184804124653) + | > grad_norm_1: tensor(189.8001, device='cuda:0') (tensor(135.8456, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 0.9807 (0.8810239147465185) + | > loader_time: 0.02 (0.013509393300650254) + + + --> TIME: 2024-06-25 08:47:03 -- STEP: 449/591 -- GLOBAL_STEP: 21725 + | > loss_disc: 2.7628705501556396 (2.702696422691602) + | > loss_disc_real_0: 0.2941455841064453 (0.2052036364685188) + | > loss_disc_real_1: 0.21719589829444885 (0.2267688644224392) + | > loss_disc_real_2: 0.23838481307029724 (0.23592101117152148) + | > loss_disc_real_3: 0.24663035571575165 (0.23505586408958137) + | > loss_disc_real_4: 0.23691952228546143 (0.23198685541184813) + | > loss_disc_real_5: 0.23959724605083466 (0.23827292031726752) + | > loss_0: 2.7628705501556396 (2.702696422691602) + | > grad_norm_0: tensor(12.3007, device='cuda:0') (tensor(10.5753, device='cuda:0')) + | > loss_gen: 1.8301825523376465 (1.928250547772261) + | > loss_kl: 2.5433576107025146 (2.533298299678982) + | > loss_feat: 3.524423360824585 (3.783628475958096) + | > loss_mel: 20.5445499420166 (20.82135872108104) + | > loss_duration: 1.3716011047363281 (1.355855241650197) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.814115524291992 (30.42239133074448) + | > grad_norm_1: tensor(77.3358, device='cuda:0') (tensor(136.5582, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 1.0198 (0.888162687254908) + | > loader_time: 0.0178 (0.013785242237864198) + + + --> TIME: 2024-06-25 08:47:30 -- STEP: 474/591 -- GLOBAL_STEP: 21750 + | > loss_disc: 2.708726406097412 (2.702919099904316) + | > loss_disc_real_0: 0.17852874100208282 (0.20503167776618828) + | > loss_disc_real_1: 0.2471216320991516 (0.22675361207643138) + | > loss_disc_real_2: 0.22703014314174652 (0.23597202289708052) + | > loss_disc_real_3: 0.2987610995769501 (0.23516767975021516) + | > loss_disc_real_4: 0.2501279413700104 (0.2320909670196505) + | > loss_disc_real_5: 0.24751679599285126 (0.23829067771947837) + | > loss_0: 2.708726406097412 (2.702919099904316) + | > grad_norm_0: tensor(8.0982, device='cuda:0') (tensor(10.5610, device='cuda:0')) + | > loss_gen: 1.8983083963394165 (1.9274598548683939) + | > loss_kl: 2.542508602142334 (2.5302992938440068) + | > loss_feat: 3.505375862121582 (3.7770947534826735) + | > loss_mel: 20.524822235107422 (20.809228808568008) + | > loss_duration: 1.3430819511413574 (1.3570401044837532) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.814096450805664 (30.401122857749712) + | > grad_norm_1: tensor(265.9892, device='cuda:0') (tensor(137.4459, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 1.0136 (0.8955621452774177) + | > loader_time: 0.0176 (0.014081411220856356) + + + --> TIME: 2024-06-25 08:47:57 -- STEP: 499/591 -- GLOBAL_STEP: 21775 + | > loss_disc: 2.7530338764190674 (2.7044033478639427) + | > loss_disc_real_0: 0.26348763704299927 (0.20539058903176224) + | > loss_disc_real_1: 0.2674367427825928 (0.22691009051814107) + | > loss_disc_real_2: 0.2734691798686981 (0.23614859318207643) + | > loss_disc_real_3: 0.25138935446739197 (0.2352085598186405) + | > loss_disc_real_4: 0.20591184496879578 (0.23213794179454833) + | > loss_disc_real_5: 0.2346557229757309 (0.2384117026486712) + | > loss_0: 2.7530338764190674 (2.7044033478639427) + | > grad_norm_0: tensor(12.9580, device='cuda:0') (tensor(10.6708, device='cuda:0')) + | > loss_gen: 1.761976957321167 (1.9266973932185967) + | > loss_kl: 2.4407715797424316 (2.5295313431886974) + | > loss_feat: 3.460134267807007 (3.7680728626633453) + | > loss_mel: 20.937519073486328 (20.800384383880075) + | > loss_duration: 1.376319408416748 (1.3583032260676902) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.976720809936523 (30.38298925273643) + | > grad_norm_1: tensor(167.5111, device='cuda:0') (tensor(136.8389, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 1.1145 (0.9040812457013944) + | > loader_time: 0.0211 (0.014388136013237412) + + + --> TIME: 2024-06-25 08:48:25 -- STEP: 524/591 -- GLOBAL_STEP: 21800 + | > loss_disc: 2.696410655975342 (2.7041639371682686) + | > loss_disc_real_0: 0.21790482103824615 (0.20501021368301098) + | > loss_disc_real_1: 0.1936100274324417 (0.22692462955494874) + | > loss_disc_real_2: 0.2144595831632614 (0.23608556352840124) + | > loss_disc_real_3: 0.2064526528120041 (0.2352852196297573) + | > loss_disc_real_4: 0.22381150722503662 (0.23211222773289863) + | > loss_disc_real_5: 0.23902177810668945 (0.23838272426305837) + | > loss_0: 2.696410655975342 (2.7041639371682686) + | > grad_norm_0: tensor(4.8802, device='cuda:0') (tensor(10.5452, device='cuda:0')) + | > loss_gen: 2.0068979263305664 (1.9261318697729184) + | > loss_kl: 2.5872385501861572 (2.527819029702486) + | > loss_feat: 3.6720917224884033 (3.7613619120066404) + | > loss_mel: 20.24233055114746 (20.790712804284723) + | > loss_duration: 1.4060132503509521 (1.3599737595965844) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.914573669433594 (30.365999418360587) + | > grad_norm_1: tensor(296.7959, device='cuda:0') (tensor(138.1846, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 1.1552 (0.9141135752656079) + | > loader_time: 0.0214 (0.014729721855571252) + + + --> TIME: 2024-06-25 08:48:56 -- STEP: 549/591 -- GLOBAL_STEP: 21825 + | > loss_disc: 2.7014083862304688 (2.7046225722370267) + | > loss_disc_real_0: 0.18614821135997772 (0.20514232649014946) + | > loss_disc_real_1: 0.2521069645881653 (0.22703183302570992) + | > loss_disc_real_2: 0.2557060122489929 (0.23611729631441322) + | > loss_disc_real_3: 0.24700845777988434 (0.23533549790825353) + | > loss_disc_real_4: 0.20657503604888916 (0.232116142827305) + | > loss_disc_real_5: 0.2398984283208847 (0.23847161640560693) + | > loss_0: 2.7014083862304688 (2.7046225722370267) + | > grad_norm_0: tensor(6.2948, device='cuda:0') (tensor(10.5279, device='cuda:0')) + | > loss_gen: 1.8322657346725464 (1.9250680805338318) + | > loss_kl: 2.3603575229644775 (2.5251188955671813) + | > loss_feat: 3.525890827178955 (3.7510175153423093) + | > loss_mel: 20.442646026611328 (20.773252143234508) + | > loss_duration: 1.3731664419174194 (1.3614457448323571) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.53432846069336 (30.335902413818136) + | > grad_norm_1: tensor(72.0490, device='cuda:0') (tensor(136.6605, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 1.309 (0.9267483000763996) + | > loader_time: 0.0233 (0.01508756109491289) + + + --> TIME: 2024-06-25 08:49:28 -- STEP: 574/591 -- GLOBAL_STEP: 21850 + | > loss_disc: 2.6755964756011963 (2.7048959740362952) + | > loss_disc_real_0: 0.16982194781303406 (0.20522793538167505) + | > loss_disc_real_1: 0.21699316799640656 (0.22699022679897962) + | > loss_disc_real_2: 0.20689044892787933 (0.23599116600245135) + | > loss_disc_real_3: 0.25863727927207947 (0.2353075411988468) + | > loss_disc_real_4: 0.22954243421554565 (0.23211746314988319) + | > loss_disc_real_5: 0.2328057885169983 (0.2384429638913284) + | > loss_0: 2.6755964756011963 (2.7048959740362952) + | > grad_norm_0: tensor(6.7406, device='cuda:0') (tensor(10.5785, device='cuda:0')) + | > loss_gen: 1.9194281101226807 (1.924478850921272) + | > loss_kl: 2.446765661239624 (2.522796892122941) + | > loss_feat: 3.5245604515075684 (3.743805509411084) + | > loss_mel: 20.68243408203125 (20.758340735884083) + | > loss_duration: 1.4057589769363403 (1.3632080089755179) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.978946685791016 (30.31263002535192) + | > grad_norm_1: tensor(34.1352, device='cuda:0') (tensor(135.8096, device='cuda:0')) + | > current_lr_0: 0.0001991019659638112 + | > current_lr_1: 0.0001991019659638112 + | > step_time: 1.272 (0.94116840088409) + | > loader_time: 0.0272 (0.015478841103743174) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003579966565395923 (-5.807775132199563e-05) + | > avg_loss_disc: 2.7258817490111005 (+0.05789938886114854) + | > avg_loss_disc_real_0: 0.26178658357326035 (+0.015894930730474904) + | > avg_loss_disc_real_1: 0.1840948551259143 (-0.04976155846676919) + | > avg_loss_disc_real_2: 0.2151194598446501 (+0.010495994953399007) + | > avg_loss_disc_real_3: 0.2317439501589917 (+0.021019545641351223) + | > avg_loss_disc_real_4: 0.19810903325993964 (-0.023688608661611016) + | > avg_loss_disc_real_5: 0.21456759692506588 (-0.01905619178680662) + | > avg_loss_0: 2.7258817490111005 (+0.05789938886114854) + | > avg_loss_gen: 1.824155754231392 (-0.08434322539796235) + | > avg_loss_kl: 2.5484054900230246 (-0.05169968402132108) + | > avg_loss_feat: 3.8411460480791457 (+0.1952296825165445) + | > avg_loss_mel: 20.89498199300563 (+0.06559047293155729) + | > avg_loss_duration: 1.3746387857071898 (-0.014730996273933705) + | > avg_loss_1: 30.483328068510016 (+0.11004606206366319) + + + > EPOCH: 38/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 08:49:58)  + + --> TIME: 2024-06-25 08:50:06 -- STEP: 8/591 -- GLOBAL_STEP: 21875 + | > loss_disc: 2.7017300128936768 (2.7062072157859802) + | > loss_disc_real_0: 0.21622557938098907 (0.20472458377480507) + | > loss_disc_real_1: 0.2109665721654892 (0.23277279920876026) + | > loss_disc_real_2: 0.21395210921764374 (0.22854869067668915) + | > loss_disc_real_3: 0.22726018726825714 (0.2319307904690504) + | > loss_disc_real_4: 0.2047663778066635 (0.23908304050564766) + | > loss_disc_real_5: 0.2168215960264206 (0.23994340933859348) + | > loss_0: 2.7017300128936768 (2.7062072157859802) + | > grad_norm_0: tensor(3.4687, device='cuda:0') (tensor(6.6776, device='cuda:0')) + | > loss_gen: 2.057753086090088 (1.9608987271785736) + | > loss_kl: 2.6840291023254395 (2.6466996669769287) + | > loss_feat: 4.163578033447266 (4.061725437641144) + | > loss_mel: 20.89580726623535 (20.931451320648193) + | > loss_duration: 1.3338087797164917 (1.3380901515483856) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.13497543334961 (30.938865661621094) + | > grad_norm_1: tensor(209.8964, device='cuda:0') (tensor(218.1347, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.7656 (0.7687766253948212) + | > loader_time: 0.0086 (0.008747071027755737) + + + --> TIME: 2024-06-25 08:50:26 -- STEP: 33/591 -- GLOBAL_STEP: 21900 + | > loss_disc: 2.77108097076416 (2.7123129440076426) + | > loss_disc_real_0: 0.23225148022174835 (0.20805992276379556) + | > loss_disc_real_1: 0.23587235808372498 (0.23063525783293176) + | > loss_disc_real_2: 0.2516270875930786 (0.2348757101730867) + | > loss_disc_real_3: 0.24603991210460663 (0.23702179798574158) + | > loss_disc_real_4: 0.2570594549179077 (0.23448046364567496) + | > loss_disc_real_5: 0.24091938138008118 (0.23897939559185144) + | > loss_0: 2.77108097076416 (2.7123129440076426) + | > grad_norm_0: tensor(9.2591, device='cuda:0') (tensor(10.2678, device='cuda:0')) + | > loss_gen: 1.736229658126831 (1.9416246378060542) + | > loss_kl: 2.4453604221343994 (2.6236587509964453) + | > loss_feat: 3.4578089714050293 (3.9050838513807817) + | > loss_mel: 20.476383209228516 (20.84214210510254) + | > loss_duration: 1.339510440826416 (1.326961622093663) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.455291748046875 (30.63947105407715) + | > grad_norm_1: tensor(56.7410, device='cuda:0') (tensor(171.8638, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.788 (0.783646027247111) + | > loader_time: 0.0094 (0.009165366490681967) + + + --> TIME: 2024-06-25 08:50:46 -- STEP: 58/591 -- GLOBAL_STEP: 21925 + | > loss_disc: 2.7253761291503906 (2.712878510869782) + | > loss_disc_real_0: 0.14804156124591827 (0.20682693003066654) + | > loss_disc_real_1: 0.24370591342449188 (0.228581417737336) + | > loss_disc_real_2: 0.23894423246383667 (0.2348500801057651) + | > loss_disc_real_3: 0.20010729134082794 (0.2359398092175352) + | > loss_disc_real_4: 0.2341160625219345 (0.23381538355144962) + | > loss_disc_real_5: 0.23105488717556 (0.2393711694869502) + | > loss_0: 2.7253761291503906 (2.712878510869782) + | > grad_norm_0: tensor(12.8066, device='cuda:0') (tensor(10.5484, device='cuda:0')) + | > loss_gen: 2.0812246799468994 (1.9405472031955062) + | > loss_kl: 2.4415483474731445 (2.596951534008157) + | > loss_feat: 3.982210874557495 (3.9037354239102067) + | > loss_mel: 21.127988815307617 (20.946805756667565) + | > loss_duration: 1.338230848312378 (1.336361095823091) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.971202850341797 (30.72440107937517) + | > grad_norm_1: tensor(151.6482, device='cuda:0') (tensor(162.5703, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.8261 (0.7932926868570263) + | > loader_time: 0.0098 (0.009431880095909381) + + + --> TIME: 2024-06-25 08:51:07 -- STEP: 83/591 -- GLOBAL_STEP: 21950 + | > loss_disc: 2.709176540374756 (2.7086942052266685) + | > loss_disc_real_0: 0.21547546982765198 (0.20686848344932118) + | > loss_disc_real_1: 0.2516523599624634 (0.2279602251498096) + | > loss_disc_real_2: 0.23181207478046417 (0.23442518369979168) + | > loss_disc_real_3: 0.21200966835021973 (0.23543746769428253) + | > loss_disc_real_4: 0.2138410359621048 (0.23383194603115678) + | > loss_disc_real_5: 0.24632322788238525 (0.2390648999127997) + | > loss_0: 2.709176540374756 (2.7086942052266685) + | > grad_norm_0: tensor(7.8242, device='cuda:0') (tensor(10.0405, device='cuda:0')) + | > loss_gen: 1.8941375017166138 (1.929911543087787) + | > loss_kl: 2.572366952896118 (2.574310205068932) + | > loss_feat: 3.793271780014038 (3.8785090733723466) + | > loss_mel: 20.671123504638672 (20.972282455628168) + | > loss_duration: 1.3606159687042236 (1.3430574069540189) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.29151725769043 (30.698070801884295) + | > grad_norm_1: tensor(57.1082, device='cuda:0') (tensor(153.0239, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.8352 (0.80143156109086) + | > loader_time: 0.0104 (0.009735739374735269) + + + --> TIME: 2024-06-25 08:51:29 -- STEP: 108/591 -- GLOBAL_STEP: 21975 + | > loss_disc: 2.7106587886810303 (2.7059075589533204) + | > loss_disc_real_0: 0.27675366401672363 (0.20668129891984993) + | > loss_disc_real_1: 0.2631377577781677 (0.22733675536734085) + | > loss_disc_real_2: 0.23161286115646362 (0.2342368564395993) + | > loss_disc_real_3: 0.2097190022468567 (0.23540472501405962) + | > loss_disc_real_4: 0.2047334909439087 (0.23288643594693254) + | > loss_disc_real_5: 0.2100192755460739 (0.23874264251854685) + | > loss_0: 2.7106587886810303 (2.7059075589533204) + | > grad_norm_0: tensor(14.8870, device='cuda:0') (tensor(10.1887, device='cuda:0')) + | > loss_gen: 1.7635490894317627 (1.929324017630683) + | > loss_kl: 2.429093599319458 (2.558905502160389) + | > loss_feat: 3.9452786445617676 (3.88031687118389) + | > loss_mel: 20.56707191467285 (20.964720796655715) + | > loss_duration: 1.387579083442688 (1.3490023800620325) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.092573165893555 (30.682269626193577) + | > grad_norm_1: tensor(58.9427, device='cuda:0') (tensor(153.5428, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.8457 (0.8103616635004679) + | > loader_time: 0.0108 (0.010123266114128957) + + + --> TIME: 2024-06-25 08:51:50 -- STEP: 133/591 -- GLOBAL_STEP: 22000 + | > loss_disc: 2.702699899673462 (2.7050266498909856) + | > loss_disc_real_0: 0.23591896891593933 (0.2063317839476399) + | > loss_disc_real_1: 0.2033890187740326 (0.22663268928689168) + | > loss_disc_real_2: 0.2355027049779892 (0.23472849509321658) + | > loss_disc_real_3: 0.24335327744483948 (0.23565482017689182) + | > loss_disc_real_4: 0.26250290870666504 (0.23303483157677757) + | > loss_disc_real_5: 0.25473082065582275 (0.23877048906975223) + | > loss_0: 2.702699899673462 (2.7050266498909856) + | > grad_norm_0: tensor(8.0314, device='cuda:0') (tensor(10.3980, device='cuda:0')) + | > loss_gen: 1.9203684329986572 (1.9327175948852884) + | > loss_kl: 2.423450469970703 (2.5430441565979685) + | > loss_feat: 3.776094913482666 (3.8836449572914526) + | > loss_mel: 21.011720657348633 (20.98177208577779) + | > loss_duration: 1.3417096138000488 (1.3503253397188688) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.473342895507812 (30.691504184464762) + | > grad_norm_1: tensor(218.9453, device='cuda:0') (tensor(152.0976, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.8292 (0.816969631309796) + | > loader_time: 0.0124 (0.010384925326010335) + + + --> TIME: 2024-06-25 08:52:12 -- STEP: 158/591 -- GLOBAL_STEP: 22025 + | > loss_disc: 2.73069167137146 (2.704192469391642) + | > loss_disc_real_0: 0.12634435296058655 (0.20592858845108672) + | > loss_disc_real_1: 0.20878413319587708 (0.22643201949098443) + | > loss_disc_real_2: 0.2542508542537689 (0.23457627045580104) + | > loss_disc_real_3: 0.2608751058578491 (0.23546835732988164) + | > loss_disc_real_4: 0.2644985020160675 (0.23290848335887812) + | > loss_disc_real_5: 0.25865641236305237 (0.2389944541869284) + | > loss_0: 2.73069167137146 (2.704192469391642) + | > grad_norm_0: tensor(15.2050, device='cuda:0') (tensor(10.3537, device='cuda:0')) + | > loss_gen: 1.9750794172286987 (1.931981722010842) + | > loss_kl: 2.4839627742767334 (2.5409195076061195) + | > loss_feat: 3.8265647888183594 (3.8737987249712402) + | > loss_mel: 21.0720157623291 (20.940268588971474) + | > loss_duration: 1.336517333984375 (1.3484958304634576) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.69413948059082 (30.635464366478256) + | > grad_norm_1: tensor(108.9950, device='cuda:0') (tensor(148.0561, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.852 (0.8251325058031685) + | > loader_time: 0.0113 (0.01065285296379765) + + + --> TIME: 2024-06-25 08:52:34 -- STEP: 183/591 -- GLOBAL_STEP: 22050 + | > loss_disc: 2.6493396759033203 (2.703870572679031) + | > loss_disc_real_0: 0.12907494604587555 (0.20583062871394914) + | > loss_disc_real_1: 0.22937263548374176 (0.22648808628809255) + | > loss_disc_real_2: 0.2146805226802826 (0.23441724031349348) + | > loss_disc_real_3: 0.24809648096561432 (0.2351481682452999) + | > loss_disc_real_4: 0.22229412198066711 (0.23281945996597164) + | > loss_disc_real_5: 0.23738117516040802 (0.23879338613624781) + | > loss_0: 2.6493396759033203 (2.703870572679031) + | > grad_norm_0: tensor(14.9657, device='cuda:0') (tensor(10.3955, device='cuda:0')) + | > loss_gen: 1.9531407356262207 (1.9301744907931553) + | > loss_kl: 2.4687564373016357 (2.533619888493272) + | > loss_feat: 4.076267719268799 (3.870253125175101) + | > loss_mel: 20.8236083984375 (20.928201383580262) + | > loss_duration: 1.35392427444458 (1.347263293188126) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.675697326660156 (30.60951215191617) + | > grad_norm_1: tensor(227.3622, device='cuda:0') (tensor(147.4846, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.8612 (0.8303522242874396) + | > loader_time: 0.0136 (0.010907178368073334) + + + --> TIME: 2024-06-25 08:52:57 -- STEP: 208/591 -- GLOBAL_STEP: 22075 + | > loss_disc: 2.6230034828186035 (2.703765512659001) + | > loss_disc_real_0: 0.1404590606689453 (0.2057371323283475) + | > loss_disc_real_1: 0.18692603707313538 (0.22638379080364338) + | > loss_disc_real_2: 0.22492775321006775 (0.23468630875532442) + | > loss_disc_real_3: 0.22286063432693481 (0.23516677900289112) + | > loss_disc_real_4: 0.24025210738182068 (0.23290640617219302) + | > loss_disc_real_5: 0.24847757816314697 (0.23877395703815496) + | > loss_0: 2.6230034828186035 (2.703765512659001) + | > grad_norm_0: tensor(9.2159, device='cuda:0') (tensor(10.3901, device='cuda:0')) + | > loss_gen: 1.9316821098327637 (1.9311637924267695) + | > loss_kl: 2.5564067363739014 (2.529732505862529) + | > loss_feat: 3.8909006118774414 (3.8681848347187042) + | > loss_mel: 20.902103424072266 (20.914376478928776) + | > loss_duration: 1.3704500198364258 (1.3466721165638704) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.65154266357422 (30.590129705575798) + | > grad_norm_1: tensor(161.4379, device='cuda:0') (tensor(150.1700, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.8758 (0.8354288442776754) + | > loader_time: 0.0126 (0.011155353142664973) + + + --> TIME: 2024-06-25 08:53:19 -- STEP: 233/591 -- GLOBAL_STEP: 22100 + | > loss_disc: 2.6884632110595703 (2.704350239217538) + | > loss_disc_real_0: 0.15783238410949707 (0.20585250250131787) + | > loss_disc_real_1: 0.2154904454946518 (0.2266796025339626) + | > loss_disc_real_2: 0.24373944103717804 (0.23441501556547925) + | > loss_disc_real_3: 0.2099546194076538 (0.2353176238582881) + | > loss_disc_real_4: 0.23400235176086426 (0.23288585296772069) + | > loss_disc_real_5: 0.21670320630073547 (0.23882020210503507) + | > loss_0: 2.6884632110595703 (2.704350239217538) + | > grad_norm_0: tensor(10.1279, device='cuda:0') (tensor(10.2568, device='cuda:0')) + | > loss_gen: 1.9587302207946777 (1.9308788356862867) + | > loss_kl: 2.498065233230591 (2.5289583953153416) + | > loss_feat: 4.06268835067749 (3.865457320929597) + | > loss_mel: 20.91456413269043 (20.913483476434134) + | > loss_duration: 1.3492207527160645 (1.3464645781742146) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.78326988220215 (30.58524252928378) + | > grad_norm_1: tensor(68.4262, device='cuda:0') (tensor(147.4249, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.8716 (0.8401974454969807) + | > loader_time: 0.0128 (0.01149491895421891) + + + --> TIME: 2024-06-25 08:53:42 -- STEP: 258/591 -- GLOBAL_STEP: 22125 + | > loss_disc: 2.6949572563171387 (2.7053981732952512) + | > loss_disc_real_0: 0.3129230737686157 (0.2065070018807585) + | > loss_disc_real_1: 0.26983338594436646 (0.22668520973403325) + | > loss_disc_real_2: 0.2583538293838501 (0.23471362582703895) + | > loss_disc_real_3: 0.25418904423713684 (0.23529893489078033) + | > loss_disc_real_4: 0.22284351289272308 (0.23281696484994518) + | > loss_disc_real_5: 0.2198483794927597 (0.2385952665593273) + | > loss_0: 2.6949572563171387 (2.7053981732952512) + | > grad_norm_0: tensor(18.9112, device='cuda:0') (tensor(10.4484, device='cuda:0')) + | > loss_gen: 1.893521785736084 (1.929988906827084) + | > loss_kl: 2.5633203983306885 (2.528313802194225) + | > loss_feat: 4.177282333374023 (3.8584005130353822) + | > loss_mel: 21.13252067565918 (20.90123220931651) + | > loss_duration: 1.352726697921753 (1.3467573934747268) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.11937141418457 (30.564692756002263) + | > grad_norm_1: tensor(87.3986, device='cuda:0') (tensor(147.4391, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.8961 (0.8456923222356989) + | > loader_time: 0.0161 (0.011817870214003918) + + + --> TIME: 2024-06-25 08:54:05 -- STEP: 283/591 -- GLOBAL_STEP: 22150 + | > loss_disc: 2.717050552368164 (2.7069840675529244) + | > loss_disc_real_0: 0.21296966075897217 (0.20673380610483694) + | > loss_disc_real_1: 0.2511533498764038 (0.22669210013778807) + | > loss_disc_real_2: 0.23866693675518036 (0.23477043137744122) + | > loss_disc_real_3: 0.22464148700237274 (0.2352196945950336) + | > loss_disc_real_4: 0.2190176546573639 (0.2328870081122267) + | > loss_disc_real_5: 0.2014865130186081 (0.2386859704875272) + | > loss_0: 2.717050552368164 (2.7069840675529244) + | > grad_norm_0: tensor(4.4374, device='cuda:0') (tensor(10.5942, device='cuda:0')) + | > loss_gen: 1.856358289718628 (1.928615082279111) + | > loss_kl: 2.4723055362701416 (2.526361366042821) + | > loss_feat: 3.8275420665740967 (3.8462434952334887) + | > loss_mel: 20.77345085144043 (20.883751070541535) + | > loss_duration: 1.342165470123291 (1.347354009378925) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.271821975708008 (30.53232496281816) + | > grad_norm_1: tensor(143.3884, device='cuda:0') (tensor(146.8835, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.8939 (0.8506364283207868) + | > loader_time: 0.0139 (0.012092522934553054) + + + --> TIME: 2024-06-25 08:54:28 -- STEP: 308/591 -- GLOBAL_STEP: 22175 + | > loss_disc: 2.6961960792541504 (2.7067717693068776) + | > loss_disc_real_0: 0.19291843473911285 (0.20664715464529276) + | > loss_disc_real_1: 0.25992047786712646 (0.22657189576269746) + | > loss_disc_real_2: 0.24989044666290283 (0.23479279411303533) + | > loss_disc_real_3: 0.23146916925907135 (0.23514541141785583) + | > loss_disc_real_4: 0.2278898060321808 (0.232871805208844) + | > loss_disc_real_5: 0.2415992021560669 (0.23876023345759936) + | > loss_0: 2.6961960792541504 (2.7067717693068776) + | > grad_norm_0: tensor(5.9363, device='cuda:0') (tensor(10.5348, device='cuda:0')) + | > loss_gen: 2.0939881801605225 (1.928185119257345) + | > loss_kl: 2.6249914169311523 (2.5248745563742396) + | > loss_feat: 4.043609619140625 (3.8404440252811876) + | > loss_mel: 21.097440719604492 (20.86784391898612) + | > loss_duration: 1.3277475833892822 (1.3479502553289582) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.187776565551758 (30.50929779201359) + | > grad_norm_1: tensor(255.4250, device='cuda:0') (tensor(145.0968, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.9185 (0.8560156148749509) + | > loader_time: 0.0165 (0.012325814018001799) + + + --> TIME: 2024-06-25 08:54:52 -- STEP: 333/591 -- GLOBAL_STEP: 22200 + | > loss_disc: 2.709933280944824 (2.706213243014822) + | > loss_disc_real_0: 0.21267099678516388 (0.20644233545831017) + | > loss_disc_real_1: 0.1880195140838623 (0.226563845266093) + | > loss_disc_real_2: 0.21851865947246552 (0.23489153783421618) + | > loss_disc_real_3: 0.23448988795280457 (0.23521799654574008) + | > loss_disc_real_4: 0.23834708333015442 (0.23281092525602462) + | > loss_disc_real_5: 0.2272314578294754 (0.2387803306092729) + | > loss_0: 2.709933280944824 (2.706213243014822) + | > grad_norm_0: tensor(4.9309, device='cuda:0') (tensor(10.3972, device='cuda:0')) + | > loss_gen: 1.8618130683898926 (1.9281177162766099) + | > loss_kl: 2.508126974105835 (2.5225830393152546) + | > loss_feat: 3.515180826187134 (3.8386560782297954) + | > loss_mel: 20.164653778076172 (20.8593115820899) + | > loss_duration: 1.3576117753982544 (1.3487597686034418) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.407386779785156 (30.497428097882427) + | > grad_norm_1: tensor(156.1576, device='cuda:0') (tensor(150.0670, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.9445 (0.8616031512125831) + | > loader_time: 0.0174 (0.01260009613838997) + + + --> TIME: 2024-06-25 08:55:16 -- STEP: 358/591 -- GLOBAL_STEP: 22225 + | > loss_disc: 2.6992123126983643 (2.7056891858244767) + | > loss_disc_real_0: 0.14961546659469604 (0.20601430433339246) + | > loss_disc_real_1: 0.23612035810947418 (0.227221257610028) + | > loss_disc_real_2: 0.21644587814807892 (0.23480526153934733) + | > loss_disc_real_3: 0.27141115069389343 (0.2353047289412115) + | > loss_disc_real_4: 0.23296386003494263 (0.2326290835132146) + | > loss_disc_real_5: 0.2199477106332779 (0.23871127878154455) + | > loss_0: 2.6992123126983643 (2.7056891858244767) + | > grad_norm_0: tensor(13.9025, device='cuda:0') (tensor(10.4247, device='cuda:0')) + | > loss_gen: 1.9380379915237427 (1.930117413651344) + | > loss_kl: 2.516392469406128 (2.5231290616136666) + | > loss_feat: 3.6477036476135254 (3.834935686441773) + | > loss_mel: 21.016210556030273 (20.86522339975368) + | > loss_duration: 1.399834394454956 (1.350027740667652) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.518177032470703 (30.503433222211275) + | > grad_norm_1: tensor(198.5415, device='cuda:0') (tensor(156.8601, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.9407 (0.8671340689312809) + | > loader_time: 0.0166 (0.012868869238059607) + + + --> TIME: 2024-06-25 08:55:41 -- STEP: 383/591 -- GLOBAL_STEP: 22250 + | > loss_disc: 2.6041858196258545 (2.7063466730690524) + | > loss_disc_real_0: 0.16348454356193542 (0.20591707813599713) + | > loss_disc_real_1: 0.22948288917541504 (0.22719109023364342) + | > loss_disc_real_2: 0.21478861570358276 (0.2349148285295571) + | > loss_disc_real_3: 0.23114541172981262 (0.23533900833005383) + | > loss_disc_real_4: 0.22365309298038483 (0.2327016128261158) + | > loss_disc_real_5: 0.24937789142131805 (0.23883663989389534) + | > loss_0: 2.6041858196258545 (2.7063466730690524) + | > grad_norm_0: tensor(6.9013, device='cuda:0') (tensor(10.2757, device='cuda:0')) + | > loss_gen: 2.1057591438293457 (1.9297181763163436) + | > loss_kl: 2.419919729232788 (2.520077770123593) + | > loss_feat: 3.9581105709075928 (3.8275104098158157) + | > loss_mel: 20.480712890625 (20.86541086699882) + | > loss_duration: 1.3706132173538208 (1.3514326698786279) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.335115432739258 (30.494149813141586) + | > grad_norm_1: tensor(99.6135, device='cuda:0') (tensor(155.8130, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.9599 (0.873376301623511) + | > loader_time: 0.0186 (0.01311481590370596) + + + --> TIME: 2024-06-25 08:56:06 -- STEP: 408/591 -- GLOBAL_STEP: 22275 + | > loss_disc: 2.7064428329467773 (2.7073054775303507) + | > loss_disc_real_0: 0.20636138319969177 (0.20650408458987288) + | > loss_disc_real_1: 0.249120831489563 (0.22710618128379187) + | > loss_disc_real_2: 0.24099212884902954 (0.23498869899149036) + | > loss_disc_real_3: 0.2570573687553406 (0.23535040886524847) + | > loss_disc_real_4: 0.23619218170642853 (0.23282098233261528) + | > loss_disc_real_5: 0.2485983520746231 (0.2389609677096208) + | > loss_0: 2.7064428329467773 (2.7073054775303507) + | > grad_norm_0: tensor(4.1859, device='cuda:0') (tensor(10.4904, device='cuda:0')) + | > loss_gen: 1.7873046398162842 (1.9290702269942153) + | > loss_kl: 2.494953155517578 (2.5183316428287346) + | > loss_feat: 3.5405383110046387 (3.820458091357175) + | > loss_mel: 20.09970474243164 (20.86107978633806) + | > loss_duration: 1.3665509223937988 (1.3521231731363366) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.289051055908203 (30.481062837675506) + | > grad_norm_1: tensor(38.3146, device='cuda:0') (tensor(151.3490, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 0.9825 (0.8796495146611155) + | > loader_time: 0.0175 (0.013391711548262944) + + + --> TIME: 2024-06-25 08:56:31 -- STEP: 433/591 -- GLOBAL_STEP: 22300 + | > loss_disc: 2.703394889831543 (2.7069138461943334) + | > loss_disc_real_0: 0.18440906703472137 (0.20624679428531723) + | > loss_disc_real_1: 0.21545402705669403 (0.22708343000389963) + | > loss_disc_real_2: 0.21474270522594452 (0.2350422379326325) + | > loss_disc_real_3: 0.22823716700077057 (0.23521568146093338) + | > loss_disc_real_4: 0.23316840827465057 (0.23282555602576938) + | > loss_disc_real_5: 0.22872117161750793 (0.23885482270915845) + | > loss_0: 2.703394889831543 (2.7069138461943334) + | > grad_norm_0: tensor(5.6908, device='cuda:0') (tensor(10.4620, device='cuda:0')) + | > loss_gen: 1.8596781492233276 (1.9278432929212999) + | > loss_kl: 2.400407552719116 (2.516686166406503) + | > loss_feat: 3.719175100326538 (3.811221429437192) + | > loss_mel: 20.200584411621094 (20.843377917247885) + | > loss_duration: 1.375239372253418 (1.3531119961110873) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.555084228515625 (30.452240725863064) + | > grad_norm_1: tensor(53.8212, device='cuda:0') (tensor(147.6369, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 1.0025 (0.8859717609150027) + | > loader_time: 0.0204 (0.013682989820872382) + + + --> TIME: 2024-06-25 08:56:57 -- STEP: 458/591 -- GLOBAL_STEP: 22325 + | > loss_disc: 2.6602840423583984 (2.7072551448272324) + | > loss_disc_real_0: 0.2396906018257141 (0.2061728958577308) + | > loss_disc_real_1: 0.2449377030134201 (0.22710180402973335) + | > loss_disc_real_2: 0.22488762438297272 (0.23505947322434212) + | > loss_disc_real_3: 0.2090936303138733 (0.23526103726250636) + | > loss_disc_real_4: 0.22185121476650238 (0.23285856475345953) + | > loss_disc_real_5: 0.23114241659641266 (0.2388431622146519) + | > loss_0: 2.6602840423583984 (2.7072551448272324) + | > grad_norm_0: tensor(9.2074, device='cuda:0') (tensor(10.5502, device='cuda:0')) + | > loss_gen: 1.9927023649215698 (1.9269426956968017) + | > loss_kl: 2.451526403427124 (2.515011122653577) + | > loss_feat: 3.8413941860198975 (3.801128319777776) + | > loss_mel: 20.522354125976562 (20.824479090594828) + | > loss_duration: 1.396580457687378 (1.3542941746232806) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.204557418823242 (30.421855322658754) + | > grad_norm_1: tensor(109.7094, device='cuda:0') (tensor(144.1128, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 1.0045 (0.893024003661876) + | > loader_time: 0.016 (0.013943160985755081) + + + --> TIME: 2024-06-25 08:57:23 -- STEP: 483/591 -- GLOBAL_STEP: 22350 + | > loss_disc: 2.731801986694336 (2.7065194210897587) + | > loss_disc_real_0: 0.2404765486717224 (0.20589610726011465) + | > loss_disc_real_1: 0.21786615252494812 (0.22696418730121715) + | > loss_disc_real_2: 0.24858532845973969 (0.2351406500144528) + | > loss_disc_real_3: 0.27281269431114197 (0.23527333063245068) + | > loss_disc_real_4: 0.2515268921852112 (0.23288428107774037) + | > loss_disc_real_5: 0.2579036355018616 (0.23891855921310914) + | > loss_0: 2.731801986694336 (2.7065194210897587) + | > grad_norm_0: tensor(7.2258, device='cuda:0') (tensor(10.4204, device='cuda:0')) + | > loss_gen: 1.8053383827209473 (1.925961384862106) + | > loss_kl: 2.5480246543884277 (2.512994419220317) + | > loss_feat: 3.698233127593994 (3.7949357269713597) + | > loss_mel: 20.515522003173828 (20.811841542182748) + | > loss_duration: 1.399489402770996 (1.3557035404702886) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.96660614013672 (30.40143650757847) + | > grad_norm_1: tensor(107.4388, device='cuda:0') (tensor(144.2734, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 1.0145 (0.8999269778684057) + | > loader_time: 0.0169 (0.014175795620272613) + + + --> TIME: 2024-06-25 08:57:51 -- STEP: 508/591 -- GLOBAL_STEP: 22375 + | > loss_disc: 2.764218330383301 (2.707007749343483) + | > loss_disc_real_0: 0.2943904995918274 (0.20605256156189236) + | > loss_disc_real_1: 0.2375565767288208 (0.2269364007286669) + | > loss_disc_real_2: 0.22809046506881714 (0.235095585513068) + | > loss_disc_real_3: 0.2181442677974701 (0.23520569183577703) + | > loss_disc_real_4: 0.22500339150428772 (0.2329031330217996) + | > loss_disc_real_5: 0.22433434426784515 (0.2390672118351685) + | > loss_0: 2.764218330383301 (2.707007749343483) + | > grad_norm_0: tensor(16.4362, device='cuda:0') (tensor(10.5207, device='cuda:0')) + | > loss_gen: 1.777554988861084 (1.9256649005600786) + | > loss_kl: 2.5741374492645264 (2.5126478587548555) + | > loss_feat: 3.5589685440063477 (3.7883956277464317) + | > loss_mel: 20.43412971496582 (20.798972490265612) + | > loss_duration: 1.361790657043457 (1.357262201196565) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.706581115722656 (30.3829429844233) + | > grad_norm_1: tensor(172.8096, device='cuda:0') (tensor(145.4637, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 1.0831 (0.9085217911427413) + | > loader_time: 0.0248 (0.014452467283864654) + + + --> TIME: 2024-06-25 08:58:19 -- STEP: 533/591 -- GLOBAL_STEP: 22400 + | > loss_disc: 2.749603271484375 (2.7079265828875188) + | > loss_disc_real_0: 0.22958742082118988 (0.20623671425607823) + | > loss_disc_real_1: 0.23882657289505005 (0.2269516948743341) + | > loss_disc_real_2: 0.25926026701927185 (0.23504250906198154) + | > loss_disc_real_3: 0.23942261934280396 (0.23542860438295868) + | > loss_disc_real_4: 0.257217139005661 (0.2328788205897383) + | > loss_disc_real_5: 0.24326466023921967 (0.23909966404397762) + | > loss_0: 2.749603271484375 (2.7079265828875188) + | > grad_norm_0: tensor(6.2774, device='cuda:0') (tensor(10.6487, device='cuda:0')) + | > loss_gen: 2.018482208251953 (1.9253766919166464) + | > loss_kl: 2.4613094329833984 (2.509857444781075) + | > loss_feat: 3.619903564453125 (3.780393152254831) + | > loss_mel: 20.413101196289062 (20.78233341696562) + | > loss_duration: 1.3748791217803955 (1.3590345197054947) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.887676239013672 (30.35699512989839) + | > grad_norm_1: tensor(87.7693, device='cuda:0') (tensor(145.2990, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 1.1968 (0.9181116217743834) + | > loader_time: 0.0178 (0.014729980679882395) + + + --> TIME: 2024-06-25 08:58:49 -- STEP: 558/591 -- GLOBAL_STEP: 22425 + | > loss_disc: 2.682857036590576 (2.7083200690566867) + | > loss_disc_real_0: 0.177631214261055 (0.20613141596904794) + | > loss_disc_real_1: 0.2061718851327896 (0.22698597997213354) + | > loss_disc_real_2: 0.2741954028606415 (0.23518520163699286) + | > loss_disc_real_3: 0.25565406680107117 (0.23553059183354874) + | > loss_disc_real_4: 0.21121525764465332 (0.23288696900170336) + | > loss_disc_real_5: 0.25493788719177246 (0.23909033068512503) + | > loss_0: 2.682857036590576 (2.7083200690566867) + | > grad_norm_0: tensor(3.9419, device='cuda:0') (tensor(10.6020, device='cuda:0')) + | > loss_gen: 1.9143627882003784 (1.9244743611222954) + | > loss_kl: 2.383134126663208 (2.5073771510927476) + | > loss_feat: 3.7134323120117188 (3.772505920847684) + | > loss_mel: 19.81365966796875 (20.76384642816359) + | > loss_duration: 1.368748664855957 (1.3607241710027054) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.193336486816406 (30.328927939083414) + | > grad_norm_1: tensor(234.9979, device='cuda:0') (tensor(145.6060, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 1.1448 (0.9292731468822794) + | > loader_time: 0.0218 (0.015024816263533828) + + + --> TIME: 2024-06-25 08:59:20 -- STEP: 583/591 -- GLOBAL_STEP: 22450 + | > loss_disc: 2.736900568008423 (2.708871065650144) + | > loss_disc_real_0: 0.18481561541557312 (0.20613600266378496) + | > loss_disc_real_1: 0.25180619955062866 (0.22700843723621827) + | > loss_disc_real_2: 0.2208099514245987 (0.23533309234456373) + | > loss_disc_real_3: 0.2304965853691101 (0.23543973461330245) + | > loss_disc_real_4: 0.22990338504314423 (0.2328763976054167) + | > loss_disc_real_5: 0.23964184522628784 (0.23906641608139254) + | > loss_0: 2.736900568008423 (2.708871065650144) + | > grad_norm_0: tensor(6.4705, device='cuda:0') (tensor(10.6046, device='cuda:0')) + | > loss_gen: 2.0041182041168213 (1.9242203000479252) + | > loss_kl: 2.4667015075683594 (2.50555289793587) + | > loss_feat: 3.574373722076416 (3.763539885165965) + | > loss_mel: 21.067378997802734 (20.751686213969368) + | > loss_duration: 1.390640139579773 (1.3625224033309904) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.503211975097656 (30.30752159759888) + | > grad_norm_1: tensor(246.7828, device='cuda:0') (tensor(147.7497, device='cuda:0')) + | > current_lr_0: 0.0001990770782180657 + | > current_lr_1: 0.0001990770782180657 + | > step_time: 1.1535 (0.9416134721419006) + | > loader_time: 0.0202 (0.01534894236569543) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0038185170356263504 (+0.0002385504702304272) + | > avg_loss_disc: 2.707187987388449 (-0.01869376162265146) + | > avg_loss_disc_real_0: 0.23510577608930303 (-0.02668080748395732) + | > avg_loss_disc_real_1: 0.2327908841853446 (+0.048696029059430296) + | > avg_loss_disc_real_2: 0.23610193203104304 (+0.02098247218639293) + | > avg_loss_disc_real_3: 0.2573651398750061 (+0.025621189716014375) + | > avg_loss_disc_real_4: 0.21646903835712594 (+0.0183600050971863) + | > avg_loss_disc_real_5: 0.24735467104201622 (+0.03278707411695034) + | > avg_loss_0: 2.707187987388449 (-0.01869376162265146) + | > avg_loss_gen: 1.9272385632738154 (+0.1030828090424234) + | > avg_loss_kl: 2.575243757126179 (+0.0268382671031544) + | > avg_loss_feat: 3.545357628071562 (-0.29578842000758376) + | > avg_loss_mel: 20.46733754746457 (-0.42764444554105907) + | > avg_loss_duration: 1.37576928290915 (+0.0011304972019601234) + | > avg_loss_1: 29.89094681435443 (-0.5923812541555868) + + > BEST MODEL : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/best_model_22458.pth + + > EPOCH: 39/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 08:59:44)  + + --> TIME: 2024-06-25 09:00:00 -- STEP: 17/591 -- GLOBAL_STEP: 22475 + | > loss_disc: 2.7311267852783203 (2.708124847973094) + | > loss_disc_real_0: 0.18517650663852692 (0.20648030323140762) + | > loss_disc_real_1: 0.183166965842247 (0.22878391953075633) + | > loss_disc_real_2: 0.20106656849384308 (0.23278883713133194) + | > loss_disc_real_3: 0.24489474296569824 (0.23806965789374182) + | > loss_disc_real_4: 0.2304568737745285 (0.23391645883812623) + | > loss_disc_real_5: 0.22573351860046387 (0.23849798563648672) + | > loss_0: 2.7311267852783203 (2.708124847973094) + | > grad_norm_0: tensor(6.3662, device='cuda:0') (tensor(7.5936, device='cuda:0')) + | > loss_gen: 1.7978451251983643 (1.9459840059280396) + | > loss_kl: 2.4953908920288086 (2.637944628210629) + | > loss_feat: 3.7680039405822754 (3.9987776840434357) + | > loss_mel: 20.968849182128906 (20.993146784165326) + | > loss_duration: 1.2598555088043213 (1.3225463278153364) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.289945602416992 (30.898399465224323) + | > grad_norm_1: tensor(256.8464, device='cuda:0') (tensor(159.7306, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.7901 (0.7757893309873694) + | > loader_time: 0.009 (0.00898889934315401) + + + --> TIME: 2024-06-25 09:00:20 -- STEP: 42/591 -- GLOBAL_STEP: 22500 + | > loss_disc: 2.7332894802093506 (2.718565197218032) + | > loss_disc_real_0: 0.25182458758354187 (0.21069588423484847) + | > loss_disc_real_1: 0.2271755188703537 (0.22638878652027675) + | > loss_disc_real_2: 0.2174183428287506 (0.23418400472118742) + | > loss_disc_real_3: 0.20887146890163422 (0.23519350091616312) + | > loss_disc_real_4: 0.2431209832429886 (0.23535918870142528) + | > loss_disc_real_5: 0.2663111984729767 (0.2412096740944045) + | > loss_0: 2.7332894802093506 (2.718565197218032) + | > grad_norm_0: tensor(12.2939, device='cuda:0') (tensor(9.9987, device='cuda:0')) + | > loss_gen: 1.9605274200439453 (1.9327513462021237) + | > loss_kl: 2.4762368202209473 (2.5959717886788503) + | > loss_feat: 3.7849090099334717 (3.891336651075454) + | > loss_mel: 21.2800235748291 (20.900977861313596) + | > loss_duration: 1.342132806777954 (1.3305606075695582) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.843828201293945 (30.65159833998907) + | > grad_norm_1: tensor(131.7141, device='cuda:0') (tensor(133.8086, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.8079 (0.7879565102713448) + | > loader_time: 0.0114 (0.009608263061160133) + + + --> TIME: 2024-06-25 09:00:41 -- STEP: 67/591 -- GLOBAL_STEP: 22525 + | > loss_disc: 2.683100700378418 (2.712543160168093) + | > loss_disc_real_0: 0.28002578020095825 (0.20976376377824527) + | > loss_disc_real_1: 0.22392629086971283 (0.2274065097766136) + | > loss_disc_real_2: 0.2340259701013565 (0.23478801704164762) + | > loss_disc_real_3: 0.28160184621810913 (0.23569339269132755) + | > loss_disc_real_4: 0.24077023565769196 (0.233755596982899) + | > loss_disc_real_5: 0.26142561435699463 (0.23956253546387402) + | > loss_0: 2.683100700378418 (2.712543160168093) + | > grad_norm_0: tensor(13.8140, device='cuda:0') (tensor(10.2654, device='cuda:0')) + | > loss_gen: 1.780665636062622 (1.9342781109596365) + | > loss_kl: 2.509525775909424 (2.5798139750067866) + | > loss_feat: 3.760864734649658 (3.861040599310576) + | > loss_mel: 20.267393112182617 (20.866899518824336) + | > loss_duration: 1.341564655303955 (1.3376525231261749) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.660015106201172 (30.57968469875962) + | > grad_norm_1: tensor(114.1481, device='cuda:0') (tensor(151.8358, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.8328 (0.7960772087324911) + | > loader_time: 0.0112 (0.00994214015220528) + + + --> TIME: 2024-06-25 09:01:02 -- STEP: 92/591 -- GLOBAL_STEP: 22550 + | > loss_disc: 2.7736098766326904 (2.707891518655031) + | > loss_disc_real_0: 0.36270570755004883 (0.20841296883704868) + | > loss_disc_real_1: 0.24583609402179718 (0.22661549079677332) + | > loss_disc_real_2: 0.24466563761234283 (0.23452550836879274) + | > loss_disc_real_3: 0.27287739515304565 (0.23506221855464188) + | > loss_disc_real_4: 0.24696867167949677 (0.23378020622160123) + | > loss_disc_real_5: 0.257318377494812 (0.23919639477263327) + | > loss_0: 2.7736098766326904 (2.707891518655031) + | > grad_norm_0: tensor(26.6325, device='cuda:0') (tensor(9.8854, device='cuda:0')) + | > loss_gen: 1.7910919189453125 (1.933495564305264) + | > loss_kl: 2.5082783699035645 (2.566407766031182) + | > loss_feat: 3.9560275077819824 (3.8723584102547686) + | > loss_mel: 20.956689834594727 (20.879033565521244) + | > loss_duration: 1.3402966260910034 (1.3456640269445335) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.552383422851562 (30.59695927993111) + | > grad_norm_1: tensor(72.5559, device='cuda:0') (tensor(134.2658, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.839 (0.8048023762910264) + | > loader_time: 0.0103 (0.010245092537092118) + + + --> TIME: 2024-06-25 09:01:23 -- STEP: 117/591 -- GLOBAL_STEP: 22575 + | > loss_disc: 2.684103012084961 (2.706902717932677) + | > loss_disc_real_0: 0.128773495554924 (0.20647019924771073) + | > loss_disc_real_1: 0.19158974289894104 (0.22686273369014773) + | > loss_disc_real_2: 0.19614090025424957 (0.23460697350848433) + | > loss_disc_real_3: 0.21115511655807495 (0.23493048026520982) + | > loss_disc_real_4: 0.2344808727502823 (0.23356920073175022) + | > loss_disc_real_5: 0.21299554407596588 (0.23865772618187797) + | > loss_0: 2.684103012084961 (2.706902717932677) + | > grad_norm_0: tensor(18.4480, device='cuda:0') (tensor(9.7440, device='cuda:0')) + | > loss_gen: 2.0900092124938965 (1.9304096393096142) + | > loss_kl: 2.5362837314605713 (2.5541780463650694) + | > loss_feat: 3.91347336769104 (3.861795663833618) + | > loss_mel: 20.562763214111328 (20.84274481096839) + | > loss_duration: 1.3142789602279663 (1.3485039625412376) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.41680908203125 (30.537632070036015) + | > grad_norm_1: tensor(49.6435, device='cuda:0') (tensor(121.9582, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.8612 (0.8112717693687506) + | > loader_time: 0.0131 (0.010412998688526635) + + + --> TIME: 2024-06-25 09:01:44 -- STEP: 142/591 -- GLOBAL_STEP: 22600 + | > loss_disc: 2.6092660427093506 (2.7068414268359335) + | > loss_disc_real_0: 0.16621966660022736 (0.20679553236130255) + | > loss_disc_real_1: 0.24207009375095367 (0.22692213438346354) + | > loss_disc_real_2: 0.22581341862678528 (0.2350972059746863) + | > loss_disc_real_3: 0.2263016700744629 (0.23504079983267986) + | > loss_disc_real_4: 0.23374053835868835 (0.23347995713563033) + | > loss_disc_real_5: 0.23080451786518097 (0.23859556344613223) + | > loss_0: 2.6092660427093506 (2.7068414268359335) + | > grad_norm_0: tensor(6.6342, device='cuda:0') (tensor(10.2253, device='cuda:0')) + | > loss_gen: 1.9243688583374023 (1.9302271159601883) + | > loss_kl: 2.455087900161743 (2.5498321929448085) + | > loss_feat: 3.9659206867218018 (3.8543807939744332) + | > loss_mel: 20.887727737426758 (20.84628172323738) + | > loss_duration: 1.364842176437378 (1.3489544425212157) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.597946166992188 (30.529676209033376) + | > grad_norm_1: tensor(83.6032, device='cuda:0') (tensor(121.6094, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.8424 (0.8166627178729421) + | > loader_time: 0.0113 (0.010561504834134806) + + + --> TIME: 2024-06-25 09:02:06 -- STEP: 167/591 -- GLOBAL_STEP: 22625 + | > loss_disc: 2.7758419513702393 (2.7065404846282775) + | > loss_disc_real_0: 0.2869287133216858 (0.2058647481535009) + | > loss_disc_real_1: 0.20602981746196747 (0.22684824315016855) + | > loss_disc_real_2: 0.25659310817718506 (0.23589203270252593) + | > loss_disc_real_3: 0.20659686625003815 (0.23522844482324795) + | > loss_disc_real_4: 0.24235911667346954 (0.23347355461049224) + | > loss_disc_real_5: 0.27065446972846985 (0.23915922597139896) + | > loss_0: 2.7758419513702393 (2.7065404846282775) + | > grad_norm_0: tensor(14.9401, device='cuda:0') (tensor(10.2140, device='cuda:0')) + | > loss_gen: 1.753676414489746 (1.9308184042662204) + | > loss_kl: 2.463300943374634 (2.545190407130532) + | > loss_feat: 3.568279504776001 (3.8552720132702123) + | > loss_mel: 20.601442337036133 (20.838049368944006) + | > loss_duration: 1.3145902156829834 (1.347480382747992) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.701290130615234 (30.516810525677162) + | > grad_norm_1: tensor(147.1161, device='cuda:0') (tensor(127.9481, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.8611 (0.8225300083617252) + | > loader_time: 0.0122 (0.010779185209445602) + + + --> TIME: 2024-06-25 09:02:28 -- STEP: 192/591 -- GLOBAL_STEP: 22650 + | > loss_disc: 2.7081727981567383 (2.7034295884271473) + | > loss_disc_real_0: 0.23683631420135498 (0.20500286897489176) + | > loss_disc_real_1: 0.1880171000957489 (0.22636711224913597) + | > loss_disc_real_2: 0.23172982037067413 (0.23573121218942106) + | > loss_disc_real_3: 0.26499414443969727 (0.23508918805358311) + | > loss_disc_real_4: 0.22477567195892334 (0.23337027283074954) + | > loss_disc_real_5: 0.2509845197200775 (0.23895513219758868) + | > loss_0: 2.7081727981567383 (2.7034295884271473) + | > grad_norm_0: tensor(6.0265, device='cuda:0') (tensor(9.8129, device='cuda:0')) + | > loss_gen: 1.8394665718078613 (1.9314094260334969) + | > loss_kl: 2.505141258239746 (2.5428050011396395) + | > loss_feat: 3.741274118423462 (3.856813815732797) + | > loss_mel: 20.602588653564453 (20.81826735536258) + | > loss_duration: 1.3455617427825928 (1.3465451250473646) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.03403091430664 (30.495840698480606) + | > grad_norm_1: tensor(151.0479, device='cuda:0') (tensor(128.6250, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.8595 (0.8279214091598989) + | > loader_time: 0.0119 (0.010975275188684457) + + + --> TIME: 2024-06-25 09:02:51 -- STEP: 217/591 -- GLOBAL_STEP: 22675 + | > loss_disc: 2.6545329093933105 (2.702689941028297) + | > loss_disc_real_0: 0.21751493215560913 (0.20479720628343967) + | > loss_disc_real_1: 0.21385811269283295 (0.22660198059224862) + | > loss_disc_real_2: 0.25647640228271484 (0.23574523936768282) + | > loss_disc_real_3: 0.23290714621543884 (0.23466987413470097) + | > loss_disc_real_4: 0.22679269313812256 (0.233299316898469) + | > loss_disc_real_5: 0.2296590954065323 (0.23890278395694522) + | > loss_0: 2.6545329093933105 (2.702689941028297) + | > grad_norm_0: tensor(4.9444, device='cuda:0') (tensor(10.0036, device='cuda:0')) + | > loss_gen: 1.8843657970428467 (1.9319859149818597) + | > loss_kl: 2.4854958057403564 (2.540170525625553) + | > loss_feat: 4.07700252532959 (3.862277417688326) + | > loss_mel: 20.986801147460938 (20.824011771909653) + | > loss_duration: 1.3173609972000122 (1.3456698980199566) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.751026153564453 (30.504115495813608) + | > grad_norm_1: tensor(314.0779, device='cuda:0') (tensor(126.8990, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.8654 (0.8330196883821269) + | > loader_time: 0.0142 (0.011228136203256064) + + + --> TIME: 2024-06-25 09:03:13 -- STEP: 242/591 -- GLOBAL_STEP: 22700 + | > loss_disc: 2.7405147552490234 (2.7014478661797274) + | > loss_disc_real_0: 0.162087544798851 (0.20512549825443707) + | > loss_disc_real_1: 0.24782241880893707 (0.2265407645628472) + | > loss_disc_real_2: 0.20624320209026337 (0.23528907651251013) + | > loss_disc_real_3: 0.209938645362854 (0.2344078255955838) + | > loss_disc_real_4: 0.23488472402095795 (0.23299633994821675) + | > loss_disc_real_5: 0.2386251538991928 (0.23840080053845714) + | > loss_0: 2.7405147552490234 (2.7014478661797274) + | > grad_norm_0: tensor(8.3158, device='cuda:0') (tensor(10.5914, device='cuda:0')) + | > loss_gen: 2.1936421394348145 (1.9364487139646671) + | > loss_kl: 2.563790798187256 (2.5410745409894573) + | > loss_feat: 3.8889009952545166 (3.8705364042077184) + | > loss_mel: 20.920488357543945 (20.824383341576443) + | > loss_duration: 1.3907569646835327 (1.346113757653669) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.957578659057617 (30.518556720954326) + | > grad_norm_1: tensor(159.0983, device='cuda:0') (tensor(129.8314, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.8722 (0.8383722078701683) + | > loader_time: 0.0129 (0.011562653809539537) + + + --> TIME: 2024-06-25 09:03:36 -- STEP: 267/591 -- GLOBAL_STEP: 22725 + | > loss_disc: 2.7576847076416016 (2.701998188254539) + | > loss_disc_real_0: 0.24342019855976105 (0.204824773838904) + | > loss_disc_real_1: 0.2862968146800995 (0.22667385989360594) + | > loss_disc_real_2: 0.26860207319259644 (0.23603560577394364) + | > loss_disc_real_3: 0.2791646420955658 (0.23502895208110525) + | > loss_disc_real_4: 0.23644933104515076 (0.23287157926443364) + | > loss_disc_real_5: 0.24846883118152618 (0.23848371391885737) + | > loss_0: 2.7576847076416016 (2.701998188254539) + | > grad_norm_0: tensor(7.1653, device='cuda:0') (tensor(10.2341, device='cuda:0')) + | > loss_gen: 1.8333446979522705 (1.9352793032756905) + | > loss_kl: 2.4596099853515625 (2.536514887649021) + | > loss_feat: 3.6692376136779785 (3.868206461717127) + | > loss_mel: 20.604122161865234 (20.80657621090778) + | > loss_duration: 1.3704442977905273 (1.3459019299303538) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.93675994873047 (30.492478781425106) + | > grad_norm_1: tensor(130.7896, device='cuda:0') (tensor(131.9805, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.9031 (0.8445784063374953) + | > loader_time: 0.0151 (0.011892568752560272) + + + --> TIME: 2024-06-25 09:04:00 -- STEP: 292/591 -- GLOBAL_STEP: 22750 + | > loss_disc: 2.6681878566741943 (2.7019786214175303) + | > loss_disc_real_0: 0.1970243752002716 (0.20445187185725122) + | > loss_disc_real_1: 0.23000356554985046 (0.22667909505432599) + | > loss_disc_real_2: 0.24970152974128723 (0.23599032690263774) + | > loss_disc_real_3: 0.26157838106155396 (0.2348689711971642) + | > loss_disc_real_4: 0.25202322006225586 (0.23299912521488045) + | > loss_disc_real_5: 0.25864216685295105 (0.23854595922852215) + | > loss_0: 2.6681878566741943 (2.7019786214175303) + | > grad_norm_0: tensor(4.5802, device='cuda:0') (tensor(10.3470, device='cuda:0')) + | > loss_gen: 1.8614263534545898 (1.9342861453147784) + | > loss_kl: 2.5485823154449463 (2.535102460482349) + | > loss_feat: 3.958587408065796 (3.8634775721863526) + | > loss_mel: 20.815053939819336 (20.79441246921069) + | > loss_duration: 1.3469057083129883 (1.3467073203766176) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.530555725097656 (30.473985965937786) + | > grad_norm_1: tensor(71.5364, device='cuda:0') (tensor(130.6524, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.9781 (0.8504607995895492) + | > loader_time: 0.0135 (0.012185709117210074) + + + --> TIME: 2024-06-25 09:04:23 -- STEP: 317/591 -- GLOBAL_STEP: 22775 + | > loss_disc: 2.7486371994018555 (2.701533844042276) + | > loss_disc_real_0: 0.23461748659610748 (0.20416466952874457) + | > loss_disc_real_1: 0.2088325172662735 (0.22661531916364133) + | > loss_disc_real_2: 0.23805300891399384 (0.23589350001105) + | > loss_disc_real_3: 0.34216323494911194 (0.23503483507535436) + | > loss_disc_real_4: 0.23756402730941772 (0.2329451456336945) + | > loss_disc_real_5: 0.25002264976501465 (0.2386134784868463) + | > loss_0: 2.7486371994018555 (2.701533844042276) + | > grad_norm_0: tensor(11.0282, device='cuda:0') (tensor(10.1710, device='cuda:0')) + | > loss_gen: 1.7849737405776978 (1.9334184371334522) + | > loss_kl: 2.575741767883301 (2.5340931092150947) + | > loss_feat: 3.6008341312408447 (3.861826801901736) + | > loss_mel: 20.078527450561523 (20.79104464437683) + | > loss_duration: 1.3431870937347412 (1.3473534689340674) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.383264541625977 (30.467736451783768) + | > grad_norm_1: tensor(155.6576, device='cuda:0') (tensor(132.5741, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.9094 (0.8555008485114163) + | > loader_time: 0.0166 (0.012462566327597445) + + + --> TIME: 2024-06-25 09:04:47 -- STEP: 342/591 -- GLOBAL_STEP: 22800 + | > loss_disc: 2.591648578643799 (2.701393134412711) + | > loss_disc_real_0: 0.182002991437912 (0.20396804478433397) + | > loss_disc_real_1: 0.23070237040519714 (0.22661491631590136) + | > loss_disc_real_2: 0.23705726861953735 (0.23594626867108875) + | > loss_disc_real_3: 0.2266920953989029 (0.23497888153930854) + | > loss_disc_real_4: 0.24229592084884644 (0.2328731225073686) + | > loss_disc_real_5: 0.23705054819583893 (0.23860777051825272) + | > loss_0: 2.591648578643799 (2.701393134412711) + | > grad_norm_0: tensor(3.4094, device='cuda:0') (tensor(10.1926, device='cuda:0')) + | > loss_gen: 2.054251194000244 (1.9332736817716856) + | > loss_kl: 2.5374536514282227 (2.530701742534748) + | > loss_feat: 3.856821060180664 (3.8570149077309503) + | > loss_mel: 20.408823013305664 (20.78361780043931) + | > loss_duration: 1.3711556196212769 (1.3485677336391644) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.22850227355957 (30.45317585147612) + | > grad_norm_1: tensor(98.3487, device='cuda:0') (tensor(130.8771, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.9312 (0.86111254998815) + | > loader_time: 0.0153 (0.012694426447327377) + + + --> TIME: 2024-06-25 09:05:11 -- STEP: 367/591 -- GLOBAL_STEP: 22825 + | > loss_disc: 2.692692756652832 (2.7012173635758248) + | > loss_disc_real_0: 0.16722333431243896 (0.20416697271026124) + | > loss_disc_real_1: 0.22248706221580505 (0.226552390678702) + | > loss_disc_real_2: 0.22102977335453033 (0.23582477469697635) + | > loss_disc_real_3: 0.21551404893398285 (0.23492410636890163) + | > loss_disc_real_4: 0.2367926985025406 (0.2326973119613585) + | > loss_disc_real_5: 0.23914840817451477 (0.23867058689003096) + | > loss_0: 2.692692756652832 (2.7012173635758248) + | > grad_norm_0: tensor(7.0223, device='cuda:0') (tensor(10.3020, device='cuda:0')) + | > loss_gen: 1.888291835784912 (1.9324263805264672) + | > loss_kl: 2.645594358444214 (2.528711377437497) + | > loss_feat: 3.7451565265655518 (3.8530985269624467) + | > loss_mel: 20.42891502380371 (20.772028842479074) + | > loss_duration: 1.3717365264892578 (1.3500887470609153) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.079694747924805 (30.43635388096282) + | > grad_norm_1: tensor(178.5905, device='cuda:0') (tensor(132.1076, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.9518 (0.8670561826846257) + | > loader_time: 0.018 (0.012953868678869931) + + + --> TIME: 2024-06-25 09:05:36 -- STEP: 392/591 -- GLOBAL_STEP: 22850 + | > loss_disc: 2.768176794052124 (2.7007320949009497) + | > loss_disc_real_0: 0.14434310793876648 (0.20414231717586517) + | > loss_disc_real_1: 0.24926665425300598 (0.22667628085734892) + | > loss_disc_real_2: 0.231308713555336 (0.23579622345159248) + | > loss_disc_real_3: 0.2295224666595459 (0.23497871336128032) + | > loss_disc_real_4: 0.21510758996009827 (0.23247725611590608) + | > loss_disc_real_5: 0.21911290287971497 (0.23850219126562683) + | > loss_0: 2.768176794052124 (2.7007320949009497) + | > grad_norm_0: tensor(21.5411, device='cuda:0') (tensor(10.4263, device='cuda:0')) + | > loss_gen: 1.9512827396392822 (1.9346524364486033) + | > loss_kl: 2.523359537124634 (2.527093563761029) + | > loss_feat: 3.520120620727539 (3.8513484512056624) + | > loss_mel: 20.379013061523438 (20.763186308802386) + | > loss_duration: 1.3835952281951904 (1.3509541795570015) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.757369995117188 (30.42723492213658) + | > grad_norm_1: tensor(133.3508, device='cuda:0') (tensor(140.6686, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.9723 (0.8732902194772449) + | > loader_time: 0.0179 (0.01322589419326004) + + + --> TIME: 2024-06-25 09:06:01 -- STEP: 417/591 -- GLOBAL_STEP: 22875 + | > loss_disc: 2.64005446434021 (2.7006037578308337) + | > loss_disc_real_0: 0.23837852478027344 (0.20426220048388702) + | > loss_disc_real_1: 0.21658936142921448 (0.22663424516038644) + | > loss_disc_real_2: 0.21100160479545593 (0.23550376542609372) + | > loss_disc_real_3: 0.2534107565879822 (0.23494131977466656) + | > loss_disc_real_4: 0.20232915878295898 (0.23258027759530275) + | > loss_disc_real_5: 0.2182958871126175 (0.23866844080763755) + | > loss_0: 2.64005446434021 (2.7006037578308337) + | > grad_norm_0: tensor(7.8367, device='cuda:0') (tensor(10.4005, device='cuda:0')) + | > loss_gen: 1.9007444381713867 (1.9333694787334195) + | > loss_kl: 2.4226369857788086 (2.5245618580056597) + | > loss_feat: 3.611314058303833 (3.8450396072378545) + | > loss_mel: 21.032941818237305 (20.740322478955303) + | > loss_duration: 1.3562549352645874 (1.351866977392054) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.323890686035156 (30.39516037774029) + | > grad_norm_1: tensor(260.0688, device='cuda:0') (tensor(139.3193, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.9823 (0.8792142330599633) + | > loader_time: 0.0159 (0.013494593919895824) + + + --> TIME: 2024-06-25 09:06:26 -- STEP: 442/591 -- GLOBAL_STEP: 22900 + | > loss_disc: 2.6232218742370605 (2.7000134277127996) + | > loss_disc_real_0: 0.1794530153274536 (0.20428890070764189) + | > loss_disc_real_1: 0.25968244671821594 (0.22647977407971118) + | > loss_disc_real_2: 0.23175427317619324 (0.2355825117203445) + | > loss_disc_real_3: 0.18079231679439545 (0.23483654458868017) + | > loss_disc_real_4: 0.20607365667819977 (0.2325103268431862) + | > loss_disc_real_5: 0.18188084661960602 (0.2385999656101158) + | > loss_0: 2.6232218742370605 (2.7000134277127996) + | > grad_norm_0: tensor(5.4978, device='cuda:0') (tensor(10.4123, device='cuda:0')) + | > loss_gen: 2.066405773162842 (1.9338417794909413) + | > loss_kl: 2.5767900943756104 (2.5229943731791282) + | > loss_feat: 4.061285018920898 (3.840256180698516) + | > loss_mel: 20.895235061645508 (20.726536185493288) + | > loss_duration: 1.3585118055343628 (1.3529503129726075) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.958227157592773 (30.376578814303713) + | > grad_norm_1: tensor(333.7896, device='cuda:0') (tensor(143.1072, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 0.996 (0.8854049816390509) + | > loader_time: 0.0198 (0.013775785044847033) + + + --> TIME: 2024-06-25 09:06:52 -- STEP: 467/591 -- GLOBAL_STEP: 22925 + | > loss_disc: 2.698275089263916 (2.699985953418806) + | > loss_disc_real_0: 0.21452122926712036 (0.20427397352980342) + | > loss_disc_real_1: 0.2223404347896576 (0.22629823387307324) + | > loss_disc_real_2: 0.2318531572818756 (0.23559784975373516) + | > loss_disc_real_3: 0.26876991987228394 (0.23500142063012214) + | > loss_disc_real_4: 0.24776965379714966 (0.23260343371545478) + | > loss_disc_real_5: 0.27614569664001465 (0.23868797180228876) + | > loss_0: 2.698275089263916 (2.699985953418806) + | > grad_norm_0: tensor(3.8319, device='cuda:0') (tensor(10.3556, device='cuda:0')) + | > loss_gen: 1.925065517425537 (1.9334234433204764) + | > loss_kl: 2.461259126663208 (2.520852665829709) + | > loss_feat: 3.7315921783447266 (3.8345799880058404) + | > loss_mel: 20.870317459106445 (20.717608711193655) + | > loss_duration: 1.3535676002502441 (1.3542569569397782) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.341800689697266 (30.36072174742084) + | > grad_norm_1: tensor(67.3554, device='cuda:0') (tensor(143.2866, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 1.0423 (0.892331377597131) + | > loader_time: 0.0206 (0.014079545243649209) + + + --> TIME: 2024-06-25 09:07:18 -- STEP: 492/591 -- GLOBAL_STEP: 22950 + | > loss_disc: 2.6861560344696045 (2.700673641228095) + | > loss_disc_real_0: 0.190723717212677 (0.20452978840567232) + | > loss_disc_real_1: 0.2396181970834732 (0.22633401655811605) + | > loss_disc_real_2: 0.22632545232772827 (0.23560838001530346) + | > loss_disc_real_3: 0.24589022994041443 (0.23506740931936396) + | > loss_disc_real_4: 0.23937945067882538 (0.23258393109450495) + | > loss_disc_real_5: 0.2454027235507965 (0.2387577337705023) + | > loss_0: 2.6861560344696045 (2.700673641228095) + | > grad_norm_0: tensor(3.3996, device='cuda:0') (tensor(10.4617, device='cuda:0')) + | > loss_gen: 1.9240363836288452 (1.9328463065430401) + | > loss_kl: 2.4923012256622314 (2.5193695358144548) + | > loss_feat: 3.6264920234680176 (3.826235621440701) + | > loss_mel: 20.792261123657227 (20.707829467649375) + | > loss_duration: 1.3903019428253174 (1.3555812055502474) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.225393295288086 (30.341862112526) + | > grad_norm_1: tensor(287.9064, device='cuda:0') (tensor(146.0601, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 1.0211 (0.8992521006886554) + | > loader_time: 0.0236 (0.014403242405837145) + + + --> TIME: 2024-06-25 09:07:46 -- STEP: 517/591 -- GLOBAL_STEP: 22975 + | > loss_disc: 2.7383220195770264 (2.7009920303780066) + | > loss_disc_real_0: 0.16297903656959534 (0.20455520088483564) + | > loss_disc_real_1: 0.22980518639087677 (0.22633554058679067) + | > loss_disc_real_2: 0.26433512568473816 (0.23572732885755468) + | > loss_disc_real_3: 0.30152782797813416 (0.23509899758854502) + | > loss_disc_real_4: 0.2175380289554596 (0.23257620439538643) + | > loss_disc_real_5: 0.24903713166713715 (0.23873432600290678) + | > loss_0: 2.7383220195770264 (2.7009920303780066) + | > grad_norm_0: tensor(9.5929, device='cuda:0') (tensor(10.4271, device='cuda:0')) + | > loss_gen: 1.9654936790466309 (1.9326505739407567) + | > loss_kl: 2.5388336181640625 (2.5171260432532945) + | > loss_feat: 3.8338782787323 (3.821203413046536) + | > loss_mel: 21.06043243408203 (20.696887202604128) + | > loss_duration: 1.405426263809204 (1.357163358703111) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.80406379699707 (30.325030564800684) + | > grad_norm_1: tensor(319.5588, device='cuda:0') (tensor(146.7031, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 1.088 (0.9076241441575426) + | > loader_time: 0.018 (0.014733649314718045) + + + --> TIME: 2024-06-25 09:08:15 -- STEP: 542/591 -- GLOBAL_STEP: 23000 + | > loss_disc: 2.613391160964966 (2.700519812502985) + | > loss_disc_real_0: 0.19724097847938538 (0.20446514534829288) + | > loss_disc_real_1: 0.1914130300283432 (0.22632193570867234) + | > loss_disc_real_2: 0.22601154446601868 (0.23566142299188456) + | > loss_disc_real_3: 0.2233268767595291 (0.23505459668121653) + | > loss_disc_real_4: 0.22683455049991608 (0.23241732192545358) + | > loss_disc_real_5: 0.25654250383377075 (0.23880652228294702) + | > loss_0: 2.613391160964966 (2.700519812502985) + | > grad_norm_0: tensor(4.4787, device='cuda:0') (tensor(10.4685, device='cuda:0')) + | > loss_gen: 2.0229737758636475 (1.9324094560753375) + | > loss_kl: 2.4935832023620605 (2.5148740399807576) + | > loss_feat: 3.840409755706787 (3.8164457175124613) + | > loss_mel: 21.43540382385254 (20.692273006228056) + | > loss_duration: 1.4236400127410889 (1.3587482809140667) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.21601104736328 (30.314750470798394) + | > grad_norm_1: tensor(137.5195, device='cuda:0') (tensor(146.1319, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 1.0926 (0.9183607915230785) + | > loader_time: 0.0236 (0.015053080896609823) + + + --> TIME: 2024-06-25 09:08:45 -- STEP: 567/591 -- GLOBAL_STEP: 23025 + | > loss_disc: 2.7046070098876953 (2.700253647681478) + | > loss_disc_real_0: 0.1393357217311859 (0.2042152472300294) + | > loss_disc_real_1: 0.26639142632484436 (0.22638872812454452) + | > loss_disc_real_2: 0.23450392484664917 (0.2355510108632806) + | > loss_disc_real_3: 0.26238447427749634 (0.23509465646806849) + | > loss_disc_real_4: 0.23174279928207397 (0.23239789537862074) + | > loss_disc_real_5: 0.24470952153205872 (0.23874740828176866) + | > loss_0: 2.7046070098876953 (2.700253647681478) + | > grad_norm_0: tensor(19.5698, device='cuda:0') (tensor(10.3398, device='cuda:0')) + | > loss_gen: 2.024646520614624 (1.9318601645700095) + | > loss_kl: 2.3880836963653564 (2.5139808722063757) + | > loss_feat: 3.523852586746216 (3.810643749893027) + | > loss_mel: 20.367143630981445 (20.68021679765543) + | > loss_duration: 1.4207613468170166 (1.36053289370562) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.7244873046875 (30.297234446073126) + | > grad_norm_1: tensor(163.9213, device='cuda:0') (tensor(148.2190, device='cuda:0')) + | > current_lr_0: 0.00019905219358328844 + | > current_lr_1: 0.00019905219358328844 + | > step_time: 1.1772 (0.9297456535292262) + | > loader_time: 0.0233 (0.015420326269913392) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036840388115416184 (-0.00013447822408473205) + | > avg_loss_disc: 2.699843817568841 (-0.007344169819607949) + | > avg_loss_disc_real_0: 0.16143370070990096 (-0.07367207537940207) + | > avg_loss_disc_real_1: 0.20536795258522034 (-0.027422931600124267) + | > avg_loss_disc_real_2: 0.2552854323640783 (+0.019183500333035258) + | > avg_loss_disc_real_3: 0.2178668167362822 (-0.039498323138723895) + | > avg_loss_disc_real_4: 0.23424199223518372 (+0.017772953878057773) + | > avg_loss_disc_real_5: 0.24762410401029789 (+0.0002694329682816632) + | > avg_loss_0: 2.699843817568841 (-0.007344169819607949) + | > avg_loss_gen: 1.832869790970011 (-0.09436877230380447) + | > avg_loss_kl: 2.530414895808443 (-0.04482886131773611) + | > avg_loss_feat: 3.837662778002151 (+0.2923051499305891) + | > avg_loss_mel: 20.92765885211052 (+0.460321304645948) + | > avg_loss_duration: 1.3908483322630538 (+0.015079049353903917) + | > avg_loss_1: 30.519454671981485 (+0.6285078576270564) + + + > EPOCH: 40/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 09:09:23)  + + --> TIME: 2024-06-25 09:09:25 -- STEP: 1/591 -- GLOBAL_STEP: 23050 + | > loss_disc: 2.693063974380493 (2.693063974380493) + | > loss_disc_real_0: 0.25212788581848145 (0.25212788581848145) + | > loss_disc_real_1: 0.2095947116613388 (0.2095947116613388) + | > loss_disc_real_2: 0.2070416510105133 (0.2070416510105133) + | > loss_disc_real_3: 0.30960986018180847 (0.30960986018180847) + | > loss_disc_real_4: 0.23128555715084076 (0.23128555715084076) + | > loss_disc_real_5: 0.24925896525382996 (0.24925896525382996) + | > loss_0: 2.693063974380493 (2.693063974380493) + | > grad_norm_0: tensor(7.1680, device='cuda:0') (tensor(7.1680, device='cuda:0')) + | > loss_gen: 1.981414556503296 (1.981414556503296) + | > loss_kl: 2.8311781883239746 (2.8311781883239746) + | > loss_feat: 3.844007730484009 (3.844007730484009) + | > loss_mel: 20.90654945373535 (20.90654945373535) + | > loss_duration: 1.3424489498138428 (1.3424489498138428) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.90559959411621 (30.90559959411621) + | > grad_norm_1: tensor(207.2825, device='cuda:0') (tensor(207.2825, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.7556 (0.755601167678833) + | > loader_time: 0.0082 (0.008168935775756836) + + + --> TIME: 2024-06-25 09:09:45 -- STEP: 26/591 -- GLOBAL_STEP: 23075 + | > loss_disc: 2.767597198486328 (2.720522697155292) + | > loss_disc_real_0: 0.31852781772613525 (0.21065180462140304) + | > loss_disc_real_1: 0.20055563747882843 (0.22652081915965447) + | > loss_disc_real_2: 0.24611353874206543 (0.2384258760855748) + | > loss_disc_real_3: 0.2703835070133209 (0.23725146169845873) + | > loss_disc_real_4: 0.2341633439064026 (0.23406086174341348) + | > loss_disc_real_5: 0.22465075552463531 (0.2420108410028311) + | > loss_0: 2.767597198486328 (2.720522697155292) + | > grad_norm_0: tensor(17.2649, device='cuda:0') (tensor(8.4178, device='cuda:0')) + | > loss_gen: 1.7253291606903076 (1.941220746590541) + | > loss_kl: 2.4537100791931152 (2.657843773181622) + | > loss_feat: 3.7323572635650635 (3.96888098349938) + | > loss_mel: 20.495718002319336 (20.87654553926908) + | > loss_duration: 1.2840330600738525 (1.3243542726223285) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.691146850585938 (30.768845264728252) + | > grad_norm_1: tensor(193.2681, device='cuda:0') (tensor(182.3823, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8179 (0.7787747108019315) + | > loader_time: 0.0103 (0.008972562276400052) + + + --> TIME: 2024-06-25 09:10:05 -- STEP: 51/591 -- GLOBAL_STEP: 23100 + | > loss_disc: 2.7730743885040283 (2.7117328597050085) + | > loss_disc_real_0: 0.24836374819278717 (0.20815693075750388) + | > loss_disc_real_1: 0.188969224691391 (0.2254170027433657) + | > loss_disc_real_2: 0.22552518546581268 (0.23495687339820115) + | > loss_disc_real_3: 0.22064539790153503 (0.2364374860828998) + | > loss_disc_real_4: 0.2515145242214203 (0.23325074975397073) + | > loss_disc_real_5: 0.2419310063123703 (0.23940031464193381) + | > loss_0: 2.7730743885040283 (2.7117328597050085) + | > grad_norm_0: tensor(10.7397, device='cuda:0') (tensor(9.9255, device='cuda:0')) + | > loss_gen: 1.9415404796600342 (1.9478468450845456) + | > loss_kl: 2.5060172080993652 (2.62447535757925) + | > loss_feat: 3.938967227935791 (3.961909612019857) + | > loss_mel: 20.083192825317383 (20.8797856873157) + | > loss_duration: 1.3423969745635986 (1.3369530532874316) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.812114715576172 (30.750970728257123) + | > grad_norm_1: tensor(113.1748, device='cuda:0') (tensor(182.2836, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8012 (0.7892321652057124) + | > loader_time: 0.0095 (0.009442680022295785) + + + --> TIME: 2024-06-25 09:10:26 -- STEP: 76/591 -- GLOBAL_STEP: 23125 + | > loss_disc: 2.6620912551879883 (2.7173925763682316) + | > loss_disc_real_0: 0.2350360006093979 (0.20883899613430626) + | > loss_disc_real_1: 0.22459319233894348 (0.22696581600528015) + | > loss_disc_real_2: 0.24781160056591034 (0.23683304731783114) + | > loss_disc_real_3: 0.2460768222808838 (0.23626753942746864) + | > loss_disc_real_4: 0.2370954304933548 (0.23397228062937134) + | > loss_disc_real_5: 0.2611910104751587 (0.24049577313034157) + | > loss_0: 2.6620912551879883 (2.7173925763682316) + | > grad_norm_0: tensor(6.7059, device='cuda:0') (tensor(9.7787, device='cuda:0')) + | > loss_gen: 1.7945892810821533 (1.9344284848163003) + | > loss_kl: 2.45945405960083 (2.596111727388281) + | > loss_feat: 3.985051155090332 (3.9227551284589266) + | > loss_mel: 20.768020629882812 (20.808433633101615) + | > loss_duration: 1.3510866165161133 (1.3409984472550849) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.358203887939453 (30.602727538660954) + | > grad_norm_1: tensor(49.4695, device='cuda:0') (tensor(167.3298, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8106 (0.7992423333619771) + | > loader_time: 0.0115 (0.009802272445277167) + + + --> TIME: 2024-06-25 09:10:47 -- STEP: 101/591 -- GLOBAL_STEP: 23150 + | > loss_disc: 2.737478733062744 (2.7104814713544183) + | > loss_disc_real_0: 0.25701311230659485 (0.20791107534181955) + | > loss_disc_real_1: 0.23315010964870453 (0.22604197426007525) + | > loss_disc_real_2: 0.2056829184293747 (0.2358659955829677) + | > loss_disc_real_3: 0.22223827242851257 (0.23510363270150553) + | > loss_disc_real_4: 0.2051028162240982 (0.23248012422925174) + | > loss_disc_real_5: 0.22512505948543549 (0.2393436147139804) + | > loss_0: 2.737478733062744 (2.7104814713544183) + | > grad_norm_0: tensor(11.4476, device='cuda:0') (tensor(10.0679, device='cuda:0')) + | > loss_gen: 2.040597915649414 (1.9377499240459781) + | > loss_kl: 2.417149782180786 (2.576031809986228) + | > loss_feat: 3.993335247039795 (3.9203857856221718) + | > loss_mel: 21.210811614990234 (20.847384443377514) + | > loss_duration: 1.360052227973938 (1.3479111595909206) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.02194595336914 (30.62946323357006) + | > grad_norm_1: tensor(347.2865, device='cuda:0') (tensor(183.6781, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8215 (0.806560752415421) + | > loader_time: 0.0105 (0.010062722876520441) + + + --> TIME: 2024-06-25 09:11:08 -- STEP: 126/591 -- GLOBAL_STEP: 23175 + | > loss_disc: 2.62015962600708 (2.7057698473097784) + | > loss_disc_real_0: 0.16686835885047913 (0.20650647379576215) + | > loss_disc_real_1: 0.21089154481887817 (0.2262634697651106) + | > loss_disc_real_2: 0.22715355455875397 (0.2361570241905394) + | > loss_disc_real_3: 0.24074575304985046 (0.2349828595206851) + | > loss_disc_real_4: 0.23350603878498077 (0.23240824847940414) + | > loss_disc_real_5: 0.23569850623607635 (0.23890108390459938) + | > loss_0: 2.62015962600708 (2.7057698473097784) + | > grad_norm_0: tensor(9.3359, device='cuda:0') (tensor(9.9821, device='cuda:0')) + | > loss_gen: 2.0511465072631836 (1.943472168748341) + | > loss_kl: 2.6620407104492188 (2.5659948871249245) + | > loss_feat: 4.173515319824219 (3.93520548608568) + | > loss_mel: 21.21448516845703 (20.86171834430998) + | > loss_duration: 1.3584566116333008 (1.3497724324937859) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.459644317626953 (30.656163412427144) + | > grad_norm_1: tensor(324.8211, device='cuda:0') (tensor(201.9899, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8418 (0.8132958847378927) + | > loader_time: 0.0107 (0.010290308604164728) + + + --> TIME: 2024-06-25 09:11:30 -- STEP: 151/591 -- GLOBAL_STEP: 23200 + | > loss_disc: 2.7019362449645996 (2.707746365212447) + | > loss_disc_real_0: 0.17611029744148254 (0.20654805340119542) + | > loss_disc_real_1: 0.1980080008506775 (0.22664282367324198) + | > loss_disc_real_2: 0.23210105299949646 (0.23657040919689154) + | > loss_disc_real_3: 0.23688967525959015 (0.2354254373256734) + | > loss_disc_real_4: 0.23230616748332977 (0.23213554928634342) + | > loss_disc_real_5: 0.2585192620754242 (0.23880008287382443) + | > loss_0: 2.7019362449645996 (2.707746365212447) + | > grad_norm_0: tensor(7.7659, device='cuda:0') (tensor(10.2808, device='cuda:0')) + | > loss_gen: 2.0611937046051025 (1.9432180022561787) + | > loss_kl: 2.3347907066345215 (2.5611355509979044) + | > loss_feat: 3.8414759635925293 (3.926124858540415) + | > loss_mel: 20.471004486083984 (20.834167152051105) + | > loss_duration: 1.3195469379425049 (1.3484406455463138) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.028011322021484 (30.61308629623312) + | > grad_norm_1: tensor(194.3302, device='cuda:0') (tensor(205.0845, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8578 (0.8209136921838419) + | > loader_time: 0.012 (0.010516652997755848) + + + --> TIME: 2024-06-25 09:11:52 -- STEP: 176/591 -- GLOBAL_STEP: 23225 + | > loss_disc: 2.691037893295288 (2.7085351496934895) + | > loss_disc_real_0: 0.17065005004405975 (0.20643567895008755) + | > loss_disc_real_1: 0.185658261179924 (0.22643111366778612) + | > loss_disc_real_2: 0.2267720103263855 (0.236767258325761) + | > loss_disc_real_3: 0.22669896483421326 (0.23535310155288738) + | > loss_disc_real_4: 0.24211016297340393 (0.23277292104268615) + | > loss_disc_real_5: 0.25438806414604187 (0.23903175697407938) + | > loss_0: 2.691037893295288 (2.7085351496934895) + | > grad_norm_0: tensor(7.9823, device='cuda:0') (tensor(10.2382, device='cuda:0')) + | > loss_gen: 1.8434339761734009 (1.9375381794842808) + | > loss_kl: 2.4732799530029297 (2.5522803339091213) + | > loss_feat: 3.6397058963775635 (3.9130881442265077) + | > loss_mel: 20.18440818786621 (20.806772513823084) + | > loss_duration: 1.312666893005371 (1.346273403276098) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 29.453495025634766 (30.55595268986442) + | > grad_norm_1: tensor(115.9179, device='cuda:0') (tensor(194.7377, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8842 (0.8270130320028826) + | > loader_time: 0.0116 (0.010775364258072594) + + + --> TIME: 2024-06-25 09:12:15 -- STEP: 201/591 -- GLOBAL_STEP: 23250 + | > loss_disc: 2.683892250061035 (2.707059061942409) + | > loss_disc_real_0: 0.210403710603714 (0.20650386380318975) + | > loss_disc_real_1: 0.24725499749183655 (0.22608686857555635) + | > loss_disc_real_2: 0.22278854250907898 (0.23637602222499562) + | > loss_disc_real_3: 0.26216360926628113 (0.23549055326637344) + | > loss_disc_real_4: 0.23842939734458923 (0.23272408522776702) + | > loss_disc_real_5: 0.2680022716522217 (0.2390183750818025) + | > loss_0: 2.683892250061035 (2.707059061942409) + | > grad_norm_0: tensor(3.6111, device='cuda:0') (tensor(9.9860, device='cuda:0')) + | > loss_gen: 2.0284016132354736 (1.9397464000170503) + | > loss_kl: 2.450813055038452 (2.5473244403725235) + | > loss_feat: 4.139031410217285 (3.919177997171582) + | > loss_mel: 20.92354965209961 (20.806457529020552) + | > loss_duration: 1.3125452995300293 (1.3455458683754087) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.854341506958008 (30.558252315616134) + | > grad_norm_1: tensor(74.4464, device='cuda:0') (tensor(193.1833, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8972 (0.8327224005514117) + | > loader_time: 0.0152 (0.011012413015413046) + + + --> TIME: 2024-06-25 09:12:37 -- STEP: 226/591 -- GLOBAL_STEP: 23275 + | > loss_disc: 2.747889995574951 (2.708568256513207) + | > loss_disc_real_0: 0.3464397192001343 (0.20712573598839537) + | > loss_disc_real_1: 0.21861672401428223 (0.22591926658575515) + | > loss_disc_real_2: 0.2703707218170166 (0.23652263895600242) + | > loss_disc_real_3: 0.22037529945373535 (0.2350362939512835) + | > loss_disc_real_4: 0.20892886817455292 (0.2328863515790585) + | > loss_disc_real_5: 0.2353539615869522 (0.23942167285533078) + | > loss_0: 2.747889995574951 (2.708568256513207) + | > grad_norm_0: tensor(24.3504, device='cuda:0') (tensor(10.0261, device='cuda:0')) + | > loss_gen: 1.7640900611877441 (1.9357889574185936) + | > loss_kl: 2.621800661087036 (2.5423341029513193) + | > loss_feat: 3.637186288833618 (3.9069238247069635) + | > loss_mel: 20.89533042907715 (20.79493866979549) + | > loss_duration: 1.3179395198822021 (1.3449356788027609) + | > amp_scaler: 512.0 (516.5309734513274) + | > loss_1: 30.236347198486328 (30.524921341279967) + | > grad_norm_1: tensor(132.0722, device='cuda:0') (tensor(188.0085, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8859 (0.8379699487601762) + | > loader_time: 0.0131 (0.011301230540317772) + + + --> TIME: 2024-06-25 09:13:00 -- STEP: 251/591 -- GLOBAL_STEP: 23300 + | > loss_disc: 2.6590704917907715 (2.7081324864193737) + | > loss_disc_real_0: 0.1863032728433609 (0.20672128082390323) + | > loss_disc_real_1: 0.24536065757274628 (0.22602631923449468) + | > loss_disc_real_2: 0.22436094284057617 (0.23634761334415452) + | > loss_disc_real_3: 0.2316540777683258 (0.234839635006935) + | > loss_disc_real_4: 0.22523444890975952 (0.23297634328978944) + | > loss_disc_real_5: 0.23176102340221405 (0.2392786493101918) + | > loss_0: 2.6590704917907715 (2.7081324864193737) + | > grad_norm_0: tensor(4.5781, device='cuda:0') (tensor(9.9939, device='cuda:0')) + | > loss_gen: 1.9487205743789673 (1.9346296298076433) + | > loss_kl: 2.547361373901367 (2.5412571886146207) + | > loss_feat: 3.939316749572754 (3.9037368440058127) + | > loss_mel: 20.299488067626953 (20.77709210156445) + | > loss_duration: 1.3478403091430664 (1.3453212909964456) + | > amp_scaler: 512.0 (516.0796812749004) + | > loss_1: 30.08272933959961 (30.50203716992382) + | > grad_norm_1: tensor(102.5073, device='cuda:0') (tensor(178.5294, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8931 (0.8435668185412646) + | > loader_time: 0.0153 (0.011598499647649635) + + + --> TIME: 2024-06-25 09:13:23 -- STEP: 276/591 -- GLOBAL_STEP: 23325 + | > loss_disc: 2.6652021408081055 (2.7085856510245283) + | > loss_disc_real_0: 0.15252074599266052 (0.20665063568647354) + | > loss_disc_real_1: 0.23943984508514404 (0.22594175995260043) + | > loss_disc_real_2: 0.25795650482177734 (0.23662469352501025) + | > loss_disc_real_3: 0.22877250611782074 (0.23505070670575334) + | > loss_disc_real_4: 0.23593008518218994 (0.23308482533995656) + | > loss_disc_real_5: 0.23890884220600128 (0.23920656054996062) + | > loss_0: 2.6652021408081055 (2.7085856510245283) + | > grad_norm_0: tensor(10.5729, device='cuda:0') (tensor(10.1889, device='cuda:0')) + | > loss_gen: 1.8763694763183594 (1.9334615330765212) + | > loss_kl: 2.5347161293029785 (2.53863437452178) + | > loss_feat: 3.9279675483703613 (3.8985506909480994) + | > loss_mel: 20.91391372680664 (20.765399179596834) + | > loss_duration: 1.3460829257965088 (1.3452798430470454) + | > amp_scaler: 512.0 (515.7101449275362) + | > loss_1: 30.599050521850586 (30.48132574385491) + | > grad_norm_1: tensor(118.7149, device='cuda:0') (tensor(176.4479, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.8908 (0.8487282101658808) + | > loader_time: 0.0159 (0.011916791183361109) + + + --> TIME: 2024-06-25 09:13:46 -- STEP: 301/591 -- GLOBAL_STEP: 23350 + | > loss_disc: 2.8260951042175293 (2.7083141732453524) + | > loss_disc_real_0: 0.13137628138065338 (0.20661795545257994) + | > loss_disc_real_1: 0.2129792869091034 (0.22595785315646683) + | > loss_disc_real_2: 0.2297898232936859 (0.2364751013905503) + | > loss_disc_real_3: 0.2578119933605194 (0.23495331411543874) + | > loss_disc_real_4: 0.24299420416355133 (0.23314066350064008) + | > loss_disc_real_5: 0.2573159635066986 (0.23924462239805647) + | > loss_0: 2.8260951042175293 (2.7083141732453524) + | > grad_norm_0: tensor(25.9271, device='cuda:0') (tensor(10.1003, device='cuda:0')) + | > loss_gen: 1.9016892910003662 (1.933595286651307) + | > loss_kl: 2.51450252532959 (2.5359229344466194) + | > loss_feat: 3.78464937210083 (3.892004054250115) + | > loss_mel: 20.624752044677734 (20.75826030870609) + | > loss_duration: 1.3639659881591797 (1.3459735922639156) + | > amp_scaler: 512.0 (515.4019933554819) + | > loss_1: 30.189559936523438 (30.465756299091733) + | > grad_norm_1: tensor(123.3282, device='cuda:0') (tensor(177.3924, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.9018 (0.8541199685727243) + | > loader_time: 0.016 (0.012172535804419026) + + + --> TIME: 2024-06-25 09:14:10 -- STEP: 326/591 -- GLOBAL_STEP: 23375 + | > loss_disc: 2.722625494003296 (2.7089312464181656) + | > loss_disc_real_0: 0.1548905372619629 (0.20676397438521035) + | > loss_disc_real_1: 0.2305794358253479 (0.2261916903630356) + | > loss_disc_real_2: 0.2405649572610855 (0.2365311381001414) + | > loss_disc_real_3: 0.19443652033805847 (0.23499711351526295) + | > loss_disc_real_4: 0.25331297516822815 (0.23314880922528133) + | > loss_disc_real_5: 0.2763790488243103 (0.23908991028377616) + | > loss_0: 2.722625494003296 (2.7089312464181656) + | > grad_norm_0: tensor(11.3041, device='cuda:0') (tensor(10.4637, device='cuda:0')) + | > loss_gen: 1.9529268741607666 (1.9341129382695157) + | > loss_kl: 2.3117411136627197 (2.5331547106701913) + | > loss_feat: 3.8469324111938477 (3.8904556101816565) + | > loss_mel: 20.534297943115234 (20.763719131610152) + | > loss_duration: 1.3651654720306396 (1.3467758074859908) + | > amp_scaler: 512.0 (515.1411042944784) + | > loss_1: 30.011062622070312 (30.468218311941698) + | > grad_norm_1: tensor(344.0804, device='cuda:0') (tensor(174.0094, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.9294 (0.8594849307112898) + | > loader_time: 0.0162 (0.012428559408597417) + + + --> TIME: 2024-06-25 09:14:34 -- STEP: 351/591 -- GLOBAL_STEP: 23400 + | > loss_disc: 2.728318691253662 (2.7089164012517704) + | > loss_disc_real_0: 0.177394300699234 (0.20636605331276217) + | > loss_disc_real_1: 0.19312477111816406 (0.2261789522619329) + | > loss_disc_real_2: 0.24016918241977692 (0.23638401106212215) + | > loss_disc_real_3: 0.20476417243480682 (0.23539054185746403) + | > loss_disc_real_4: 0.2370002418756485 (0.23318713185963807) + | > loss_disc_real_5: 0.22817982733249664 (0.23915341241746887) + | > loss_0: 2.728318691253662 (2.7089164012517704) + | > grad_norm_0: tensor(5.8016, device='cuda:0') (tensor(10.2865, device='cuda:0')) + | > loss_gen: 1.9388089179992676 (1.9315195168525066) + | > loss_kl: 2.4359426498413086 (2.528876947881151) + | > loss_feat: 3.5739047527313232 (3.8766569187838127) + | > loss_mel: 20.326608657836914 (20.745234752991962) + | > loss_duration: 1.3118059635162354 (1.3479078423942943) + | > amp_scaler: 512.0 (514.9173789173789) + | > loss_1: 29.58707046508789 (30.430196099131877) + | > grad_norm_1: tensor(230.0315, device='cuda:0') (tensor(171.2774, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.9659 (0.865280169027823) + | > loader_time: 0.0164 (0.012712268068579865) + + + --> TIME: 2024-06-25 09:14:58 -- STEP: 376/591 -- GLOBAL_STEP: 23425 + | > loss_disc: 2.725536584854126 (2.7073690510810713) + | > loss_disc_real_0: 0.3081382215023041 (0.20636832716062348) + | > loss_disc_real_1: 0.2579967975616455 (0.226003262392701) + | > loss_disc_real_2: 0.2219381183385849 (0.23625671304762363) + | > loss_disc_real_3: 0.2465430200099945 (0.23544951239006318) + | > loss_disc_real_4: 0.24624037742614746 (0.23291789030933635) + | > loss_disc_real_5: 0.24644023180007935 (0.23892034558539696) + | > loss_0: 2.725536584854126 (2.7073690510810713) + | > grad_norm_0: tensor(17.0045, device='cuda:0') (tensor(10.3424, device='cuda:0')) + | > loss_gen: 1.8778501749038696 (1.932752604180194) + | > loss_kl: 2.514798402786255 (2.5266865878663167) + | > loss_feat: 3.9006612300872803 (3.8759952593357005) + | > loss_mel: 20.430912017822266 (20.741886306316324) + | > loss_duration: 1.38962721824646 (1.3496787411735418) + | > amp_scaler: 512.0 (514.7234042553193) + | > loss_1: 30.113847732543945 (30.426999589230153) + | > grad_norm_1: tensor(104.2613, device='cuda:0') (tensor(176.7560, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.938 (0.8708835865588899) + | > loader_time: 0.0192 (0.012965743211989705) + + + --> TIME: 2024-06-25 09:15:23 -- STEP: 401/591 -- GLOBAL_STEP: 23450 + | > loss_disc: 2.7262885570526123 (2.7074972078984536) + | > loss_disc_real_0: 0.273704469203949 (0.20619145502101752) + | > loss_disc_real_1: 0.21618589758872986 (0.22609095591262093) + | > loss_disc_real_2: 0.2410368174314499 (0.23639552127987964) + | > loss_disc_real_3: 0.22359216213226318 (0.23557955286746607) + | > loss_disc_real_4: 0.25748249888420105 (0.2329701464669663) + | > loss_disc_real_5: 0.2563828229904175 (0.23894688955268956) + | > loss_0: 2.7262885570526123 (2.7074972078984536) + | > grad_norm_0: tensor(13.5501, device='cuda:0') (tensor(10.3545, device='cuda:0')) + | > loss_gen: 1.859403371810913 (1.9333334528002655) + | > loss_kl: 2.5613815784454346 (2.5241293734743127) + | > loss_feat: 3.6573221683502197 (3.8723937067902297) + | > loss_mel: 20.651931762695312 (20.736528282450944) + | > loss_duration: 1.369513750076294 (1.3505844139398782) + | > amp_scaler: 512.0 (514.5536159600999) + | > loss_1: 30.09955406188965 (30.416969308829366) + | > grad_norm_1: tensor(111.2192, device='cuda:0') (tensor(181.0019, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 1.0407 (0.8774101395262149) + | > loader_time: 0.0185 (0.013235463763115707) + + + --> TIME: 2024-06-25 09:15:48 -- STEP: 426/591 -- GLOBAL_STEP: 23475 + | > loss_disc: 2.6886048316955566 (2.707876414200512) + | > loss_disc_real_0: 0.14432638883590698 (0.20622015599283813) + | > loss_disc_real_1: 0.20990419387817383 (0.2263198414458915) + | > loss_disc_real_2: 0.21700860559940338 (0.23621544413303544) + | > loss_disc_real_3: 0.23039159178733826 (0.23543933404723244) + | > loss_disc_real_4: 0.22910796105861664 (0.23301304292650848) + | > loss_disc_real_5: 0.2399233877658844 (0.2388261049076425) + | > loss_0: 2.6886048316955566 (2.707876414200512) + | > grad_norm_0: tensor(15.0168, device='cuda:0') (tensor(10.5413, device='cuda:0')) + | > loss_gen: 2.0065829753875732 (1.9335594792880921) + | > loss_kl: 2.4345648288726807 (2.5213696352193065) + | > loss_feat: 3.9695169925689697 (3.8641669168158876) + | > loss_mel: 20.724355697631836 (20.72170211899451) + | > loss_duration: 1.389078974723816 (1.351425196643167) + | > amp_scaler: 512.0 (514.4037558685446) + | > loss_1: 30.524097442626953 (30.392223416359773) + | > grad_norm_1: tensor(379.7491, device='cuda:0') (tensor(182.6638, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.9928 (0.884157034712778) + | > loader_time: 0.0195 (0.013520186495892877) + + + --> TIME: 2024-06-25 09:16:14 -- STEP: 451/591 -- GLOBAL_STEP: 23500 + | > loss_disc: 2.6469545364379883 (2.7079071232061978) + | > loss_disc_real_0: 0.16235598921775818 (0.20603427276048322) + | > loss_disc_real_1: 0.23027446866035461 (0.2265240694427173) + | > loss_disc_real_2: 0.23633038997650146 (0.2363304803746767) + | > loss_disc_real_3: 0.1918656975030899 (0.23544238352458388) + | > loss_disc_real_4: 0.20472830533981323 (0.23303753788481796) + | > loss_disc_real_5: 0.2311648577451706 (0.23883394391467988) + | > loss_0: 2.6469545364379883 (2.7079071232061978) + | > grad_norm_0: tensor(8.1250, device='cuda:0') (tensor(10.3676, device='cuda:0')) + | > loss_gen: 1.8880388736724854 (1.931966749633231) + | > loss_kl: 2.4666078090667725 (2.518927742266603) + | > loss_feat: 3.626434564590454 (3.854970508034108) + | > loss_mel: 19.520723342895508 (20.709561100555916) + | > loss_duration: 1.3745458126068115 (1.352452293467892) + | > amp_scaler: 512.0 (514.270509977827) + | > loss_1: 28.87635040283203 (30.367878429641216) + | > grad_norm_1: tensor(260.0470, device='cuda:0') (tensor(181.4080, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 0.9998 (0.8909522958446765) + | > loader_time: 0.0152 (0.013795357850068423) + + + --> TIME: 2024-06-25 09:16:41 -- STEP: 476/591 -- GLOBAL_STEP: 23525 + | > loss_disc: 2.6397674083709717 (2.707481243530243) + | > loss_disc_real_0: 0.19061030447483063 (0.2061443007825052) + | > loss_disc_real_1: 0.22838155925273895 (0.22630203818949332) + | > loss_disc_real_2: 0.2303447127342224 (0.23619360461079775) + | > loss_disc_real_3: 0.26780781149864197 (0.23536157279330142) + | > loss_disc_real_4: 0.20484110713005066 (0.23299016362103095) + | > loss_disc_real_5: 0.20808179676532745 (0.23887151435894124) + | > loss_0: 2.6397674083709717 (2.707481243530243) + | > grad_norm_0: tensor(3.8608, device='cuda:0') (tensor(10.4762, device='cuda:0')) + | > loss_gen: 1.9671313762664795 (1.9319809081674624) + | > loss_kl: 2.45873761177063 (2.5154791058612482) + | > loss_feat: 3.9021658897399902 (3.850256543700435) + | > loss_mel: 20.439170837402344 (20.701743234105493) + | > loss_duration: 1.3550094366073608 (1.3535596673228163) + | > amp_scaler: 512.0 (514.1512605042014) + | > loss_1: 30.122215270996094 (30.353019485954476) + | > grad_norm_1: tensor(119.4591, device='cuda:0') (tensor(179.6989, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 1.2203 (0.8984923928725619) + | > loader_time: 0.0203 (0.01410384488706829) + + + --> TIME: 2024-06-25 09:17:09 -- STEP: 501/591 -- GLOBAL_STEP: 23550 + | > loss_disc: 2.7215335369110107 (2.707196343682723) + | > loss_disc_real_0: 0.19742006063461304 (0.2060405485436112) + | > loss_disc_real_1: 0.2511480152606964 (0.22632348608827876) + | > loss_disc_real_2: 0.2802719473838806 (0.23617214663418942) + | > loss_disc_real_3: 0.23117320239543915 (0.23539415474780306) + | > loss_disc_real_4: 0.25881102681159973 (0.23290787717539393) + | > loss_disc_real_5: 0.24252630770206451 (0.23885776361305558) + | > loss_0: 2.7215335369110107 (2.707196343682723) + | > grad_norm_0: tensor(6.7517, device='cuda:0') (tensor(10.3070, device='cuda:0')) + | > loss_gen: 1.8854117393493652 (1.9312807640391672) + | > loss_kl: 2.3068699836730957 (2.512552406497583) + | > loss_feat: 3.5680367946624756 (3.845832058055672) + | > loss_mel: 19.845836639404297 (20.68846678210354) + | > loss_duration: 1.3902251720428467 (1.3548404137769385) + | > amp_scaler: 512.0 (514.0439121756483) + | > loss_1: 28.996379852294922 (30.332972452312173) + | > grad_norm_1: tensor(74.8006, device='cuda:0') (tensor(175.5139, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 1.1072 (0.9083507636825957) + | > loader_time: 0.0216 (0.01440312905225925) + + + --> TIME: 2024-06-25 09:17:38 -- STEP: 526/591 -- GLOBAL_STEP: 23575 + | > loss_disc: 2.690739154815674 (2.7070233182762076) + | > loss_disc_real_0: 0.15009720623493195 (0.2058140192563316) + | > loss_disc_real_1: 0.2038026601076126 (0.226271745114952) + | > loss_disc_real_2: 0.2549038529396057 (0.23615098472670457) + | > loss_disc_real_3: 0.24841520190238953 (0.23534631777285622) + | > loss_disc_real_4: 0.20942868292331696 (0.23287936985606023) + | > loss_disc_real_5: 0.23341074585914612 (0.23891654100708185) + | > loss_0: 2.690739154815674 (2.7070233182762076) + | > grad_norm_0: tensor(7.4745, device='cuda:0') (tensor(10.2243, device='cuda:0')) + | > loss_gen: 2.0176844596862793 (1.9309494794548239) + | > loss_kl: 2.4398910999298096 (2.51039269532541) + | > loss_feat: 3.803971529006958 (3.8397664856094824) + | > loss_mel: 20.581933975219727 (20.671702033213357) + | > loss_duration: 1.3774302005767822 (1.3565316485814725) + | > amp_scaler: 512.0 (513.9467680608361) + | > loss_1: 30.220911026000977 (30.309342377086104) + | > grad_norm_1: tensor(45.9952, device='cuda:0') (tensor(171.4703, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 1.1504 (0.9202019755831237) + | > loader_time: 0.0239 (0.014742060759221645) + + + --> TIME: 2024-06-25 09:18:09 -- STEP: 551/591 -- GLOBAL_STEP: 23600 + | > loss_disc: 2.7543790340423584 (2.7082319155795194) + | > loss_disc_real_0: 0.22365935146808624 (0.2061996674013008) + | > loss_disc_real_1: 0.253469854593277 (0.22636573930725645) + | > loss_disc_real_2: 0.25572651624679565 (0.23612266906052015) + | > loss_disc_real_3: 0.26104244589805603 (0.2355023270942771) + | > loss_disc_real_4: 0.2467976063489914 (0.2331208205374529) + | > loss_disc_real_5: 0.2534182369709015 (0.23893895942697507) + | > loss_0: 2.7543790340423584 (2.7082319155795194) + | > grad_norm_0: tensor(6.8051, device='cuda:0') (tensor(10.4517, device='cuda:0')) + | > loss_gen: 1.773560643196106 (1.9300916445882697) + | > loss_kl: 2.5435454845428467 (2.5075918152630865) + | > loss_feat: 3.2101407051086426 (3.829958659118403) + | > loss_mel: 19.865808486938477 (20.658041990387478) + | > loss_duration: 1.3962548971176147 (1.358032284544514) + | > amp_scaler: 512.0 (513.8584392014515) + | > loss_1: 28.7893123626709 (30.28371640601738) + | > grad_norm_1: tensor(74.7913, device='cuda:0') (tensor(167.7081, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 1.2074 (0.9332981386548161) + | > loader_time: 0.0234 (0.015112179809819548) + + + --> TIME: 2024-06-25 09:18:42 -- STEP: 576/591 -- GLOBAL_STEP: 23625 + | > loss_disc: 2.6647331714630127 (2.7085563391447085) + | > loss_disc_real_0: 0.19648680090904236 (0.20600396875912946) + | > loss_disc_real_1: 0.22085699439048767 (0.2265290228339533) + | > loss_disc_real_2: 0.250217080116272 (0.23625940164654619) + | > loss_disc_real_3: 0.23341940343379974 (0.23535120458756056) + | > loss_disc_real_4: 0.2390611320734024 (0.233209068266054) + | > loss_disc_real_5: 0.22486723959445953 (0.2389272509349717) + | > loss_0: 2.6647331714630127 (2.7085563391447085) + | > grad_norm_0: tensor(3.7519, device='cuda:0') (tensor(10.3505, device='cuda:0')) + | > loss_gen: 1.87418532371521 (1.928269914040963) + | > loss_kl: 2.4445509910583496 (2.505688654051886) + | > loss_feat: 3.853715181350708 (3.8189842651287713) + | > loss_mel: 20.465551376342773 (20.640905158387316) + | > loss_duration: 1.4092438220977783 (1.3598066711177428) + | > amp_scaler: 512.0 (513.7777777777771) + | > loss_1: 30.0472469329834 (30.25365467535125) + | > grad_norm_1: tensor(328.6008, device='cuda:0') (tensor(165.6169, device='cuda:0')) + | > current_lr_0: 0.0001990273120590905 + | > current_lr_1: 0.0001990273120590905 + | > step_time: 1.3118 (0.9481381024751399) + | > loader_time: 0.0222 (0.015531589587529501) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036573105670036155 (-2.6728244538002863e-05) + | > avg_loss_disc: 2.6810558603164996 (-0.018787957252341503) + | > avg_loss_disc_real_0: 0.24085123012674617 (+0.07941752941684521) + | > avg_loss_disc_real_1: 0.21209325467018372 (+0.006725302084963386) + | > avg_loss_disc_real_2: 0.24127086775099976 (-0.01401456461307854) + | > avg_loss_disc_real_3: 0.23195212160019166 (+0.014085304863909465) + | > avg_loss_disc_real_4: 0.246288315729892 (+0.012046323494708272) + | > avg_loss_disc_real_5: 0.23882097354594697 (-0.008803130464350911) + | > avg_loss_0: 2.6810558603164996 (-0.018787957252341503) + | > avg_loss_gen: 1.9404332181240649 (+0.10756342715405398) + | > avg_loss_kl: 2.573281501201873 (+0.042866605393430124) + | > avg_loss_feat: 3.7020219792710973 (-0.13564079873105372) + | > avg_loss_mel: 20.639343992192696 (-0.288314859917822) + | > avg_loss_duration: 1.3764182278450499 (-0.014430104418003964) + | > avg_loss_1: 30.23149879942549 (-0.28795587255599386) + + + > EPOCH: 41/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 09:19:09)  + + --> TIME: 2024-06-25 09:19:19 -- STEP: 10/591 -- GLOBAL_STEP: 23650 + | > loss_disc: 2.6811602115631104 (2.6700136184692385) + | > loss_disc_real_0: 0.13188567757606506 (0.18867552280426025) + | > loss_disc_real_1: 0.2225799858570099 (0.2226994127035141) + | > loss_disc_real_2: 0.229194775223732 (0.22807398289442063) + | > loss_disc_real_3: 0.16310705244541168 (0.2341119334101677) + | > loss_disc_real_4: 0.22201454639434814 (0.23201008886098862) + | > loss_disc_real_5: 0.2665906250476837 (0.24048147350549698) + | > loss_0: 2.6811602115631104 (2.6700136184692385) + | > grad_norm_0: tensor(15.9643, device='cuda:0') (tensor(8.9364, device='cuda:0')) + | > loss_gen: 2.2316761016845703 (2.0000192284584046) + | > loss_kl: 2.6752126216888428 (2.6911044836044313) + | > loss_feat: 4.137205123901367 (4.174381995201111) + | > loss_mel: 20.84900665283203 (20.85738887786865) + | > loss_duration: 1.3409950733184814 (1.3285632967948913) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.23409652709961 (31.05145835876465) + | > grad_norm_1: tensor(50.4759, device='cuda:0') (tensor(139.8052, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.776 (0.7683455944061279) + | > loader_time: 0.0088 (0.008469605445861816) + + + --> TIME: 2024-06-25 09:19:39 -- STEP: 35/591 -- GLOBAL_STEP: 23675 + | > loss_disc: 2.711277484893799 (2.712187719345093) + | > loss_disc_real_0: 0.24398921430110931 (0.20737627595663072) + | > loss_disc_real_1: 0.2452356070280075 (0.22861198910645075) + | > loss_disc_real_2: 0.22662335634231567 (0.23531784117221832) + | > loss_disc_real_3: 0.24856971204280853 (0.23621302672794886) + | > loss_disc_real_4: 0.2290416657924652 (0.2348130145243236) + | > loss_disc_real_5: 0.2346787005662918 (0.2406721362045833) + | > loss_0: 2.711277484893799 (2.712187719345093) + | > grad_norm_0: tensor(10.2604, device='cuda:0') (tensor(11.9166, device='cuda:0')) + | > loss_gen: 1.7662149667739868 (1.9596381051199776) + | > loss_kl: 2.474560022354126 (2.633155141557966) + | > loss_feat: 3.8505287170410156 (4.027131135123117) + | > loss_mel: 20.734397888183594 (20.771488026210243) + | > loss_duration: 1.316399097442627 (1.3247236251831054) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.142099380493164 (30.716136387416295) + | > grad_norm_1: tensor(253.5720, device='cuda:0') (tensor(141.6632, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.8072 (0.7826243264334543) + | > loader_time: 0.0098 (0.00918769155229841) + + + --> TIME: 2024-06-25 09:19:59 -- STEP: 60/591 -- GLOBAL_STEP: 23700 + | > loss_disc: 2.7056121826171875 (2.701997725168864) + | > loss_disc_real_0: 0.16850200295448303 (0.20332203569511573) + | > loss_disc_real_1: 0.2062283754348755 (0.22664503703514735) + | > loss_disc_real_2: 0.22099117934703827 (0.23659354497989019) + | > loss_disc_real_3: 0.2534061372280121 (0.23497672453522683) + | > loss_disc_real_4: 0.25171056389808655 (0.23400421316425005) + | > loss_disc_real_5: 0.23793141543865204 (0.23882619788249335) + | > loss_0: 2.7056121826171875 (2.701997725168864) + | > grad_norm_0: tensor(6.9853, device='cuda:0') (tensor(10.1724, device='cuda:0')) + | > loss_gen: 2.000671863555908 (1.9525126020113628) + | > loss_kl: 2.6917638778686523 (2.6056708614031474) + | > loss_feat: 3.918386697769165 (3.9796767870585126) + | > loss_mel: 20.945222854614258 (20.71517086029052) + | > loss_duration: 1.3794448375701904 (1.3341012318929037) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.935489654541016 (30.587132453918457) + | > grad_norm_1: tensor(146.5693, device='cuda:0') (tensor(146.2016, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.8133 (0.7937396248181661) + | > loader_time: 0.0108 (0.009654478232065836) + + + --> TIME: 2024-06-25 09:20:20 -- STEP: 85/591 -- GLOBAL_STEP: 23725 + | > loss_disc: 2.750831365585327 (2.7052914731642774) + | > loss_disc_real_0: 0.2511412799358368 (0.20364454844418695) + | > loss_disc_real_1: 0.2142600268125534 (0.22742550022461835) + | > loss_disc_real_2: 0.24314068257808685 (0.23561559144188376) + | > loss_disc_real_3: 0.2982197403907776 (0.23573684920282925) + | > loss_disc_real_4: 0.21463313698768616 (0.23407622838721556) + | > loss_disc_real_5: 0.21966391801834106 (0.23883469209951513) + | > loss_0: 2.750831365585327 (2.7052914731642774) + | > grad_norm_0: tensor(9.6238, device='cuda:0') (tensor(10.9306, device='cuda:0')) + | > loss_gen: 1.9359216690063477 (1.9446897983551026) + | > loss_kl: 2.6516079902648926 (2.5879857119391954) + | > loss_feat: 3.918588876724243 (3.9498046959147732) + | > loss_mel: 21.3170166015625 (20.765857068230112) + | > loss_duration: 1.3940314054489136 (1.3414039681939516) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.217166900634766 (30.589741403916303) + | > grad_norm_1: tensor(186.6035, device='cuda:0') (tensor(134.4009, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.831 (0.8043022408204921) + | > loader_time: 0.0105 (0.009923236510332894) + + + --> TIME: 2024-06-25 09:20:42 -- STEP: 110/591 -- GLOBAL_STEP: 23750 + | > loss_disc: 2.722750425338745 (2.7074318994175304) + | > loss_disc_real_0: 0.2547309994697571 (0.20452675081112168) + | > loss_disc_real_1: 0.23828862607479095 (0.22760315388441085) + | > loss_disc_real_2: 0.25765278935432434 (0.23521362353454936) + | > loss_disc_real_3: 0.2713092267513275 (0.23723007169636814) + | > loss_disc_real_4: 0.2309148609638214 (0.23337670144709674) + | > loss_disc_real_5: 0.23128563165664673 (0.23844162157990717) + | > loss_0: 2.722750425338745 (2.7074318994175304) + | > grad_norm_0: tensor(9.0379, device='cuda:0') (tensor(10.8920, device='cuda:0')) + | > loss_gen: 1.8496754169464111 (1.940137714689428) + | > loss_kl: 2.5701568126678467 (2.568205772746694) + | > loss_feat: 3.82599139213562 (3.9398916808041657) + | > loss_mel: 20.63514518737793 (20.782741321216918) + | > loss_duration: 1.355705976486206 (1.347066467458551) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 30.236675262451172 (30.578043018687854) + | > grad_norm_1: tensor(186.2364, device='cuda:0') (tensor(147.7751, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.8303 (0.810733064738187) + | > loader_time: 0.0106 (0.010126612403176048) + + + --> TIME: 2024-06-25 09:21:03 -- STEP: 135/591 -- GLOBAL_STEP: 23775 + | > loss_disc: 2.6669113636016846 (2.7068920859584105) + | > loss_disc_real_0: 0.15816712379455566 (0.2047868831842034) + | > loss_disc_real_1: 0.25349318981170654 (0.22694346220405012) + | > loss_disc_real_2: 0.2230425924062729 (0.23530962886633697) + | > loss_disc_real_3: 0.24869655072689056 (0.23693102200826008) + | > loss_disc_real_4: 0.22301769256591797 (0.2331384484414701) + | > loss_disc_real_5: 0.24564124643802643 (0.2387592433779328) + | > loss_0: 2.6669113636016846 (2.7068920859584105) + | > grad_norm_0: tensor(9.5115, device='cuda:0') (tensor(10.5798, device='cuda:0')) + | > loss_gen: 1.9795796871185303 (1.9412010351816813) + | > loss_kl: 2.527717113494873 (2.5546189926288756) + | > loss_feat: 4.136227130889893 (3.945345894495646) + | > loss_mel: 21.186485290527344 (20.811361030296034) + | > loss_duration: 1.3287335634231567 (1.3483380644409741) + | > amp_scaler: 512.0 (512.0) + | > loss_1: 31.158742904663086 (30.600865074440286) + | > grad_norm_1: tensor(368.1545, device='cuda:0') (tensor(165.9326, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.8572 (0.8167053593529595) + | > loader_time: 0.0124 (0.010359921278776945) + + + --> TIME: 2024-06-25 09:21:25 -- STEP: 160/591 -- GLOBAL_STEP: 23800 + | > loss_disc: 2.736907958984375 (2.712329770624637) + | > loss_disc_real_0: 0.1268375962972641 (0.2054115253034979) + | > loss_disc_real_1: 0.31283098459243774 (0.23295784713700413) + | > loss_disc_real_2: 0.23407652974128723 (0.23620167057961225) + | > loss_disc_real_3: 0.23160266876220703 (0.23655224917456508) + | > loss_disc_real_4: 0.19532139599323273 (0.23175363428890705) + | > loss_disc_real_5: 0.2568226158618927 (0.23823981722816825) + | > loss_0: 2.736907958984375 (2.712329770624637) + | > grad_norm_0: tensor(21.2098, device='cuda:0') (tensor(11.6232, device='cuda:0')) + | > loss_gen: 1.9511070251464844 (1.9670798294246197) + | > loss_kl: 2.443192481994629 (2.542691557109356) + | > loss_feat: 3.955507278442383 (3.9779698327183723) + | > loss_mel: 21.865442276000977 (20.836542332172385) + | > loss_duration: 1.3316153287887573 (1.3468645602464677) + | > amp_scaler: 256.0 (476.8) + | > loss_1: 31.546863555908203 (30.671148204803465) + | > grad_norm_1: tensor(490.2785, device='cuda:0') (tensor(204.5425, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.8681 (0.8231597691774368) + | > loader_time: 0.0113 (0.01066190004348755) + + + --> TIME: 2024-06-25 09:21:47 -- STEP: 185/591 -- GLOBAL_STEP: 23825 + | > loss_disc: 2.6727614402770996 (2.7094487512433845) + | > loss_disc_real_0: 0.19457492232322693 (0.20505558216088526) + | > loss_disc_real_1: 0.20117051899433136 (0.23135647032711956) + | > loss_disc_real_2: 0.23687072098255157 (0.23540893583684355) + | > loss_disc_real_3: 0.2552383244037628 (0.2359759803559329) + | > loss_disc_real_4: 0.21728567779064178 (0.2318280322326196) + | > loss_disc_real_5: 0.21583642065525055 (0.2383642638051832) + | > loss_0: 2.6727614402770996 (2.7094487512433845) + | > grad_norm_0: tensor(4.5839, device='cuda:0') (tensor(11.1343, device='cuda:0')) + | > loss_gen: 1.923953652381897 (1.9622596914703783) + | > loss_kl: 2.631300926208496 (2.538007147247726) + | > loss_feat: 4.006866455078125 (3.970292231843278) + | > loss_mel: 20.867023468017578 (20.84092276289655) + | > loss_duration: 1.3649144172668457 (1.3457071020796492) + | > amp_scaler: 256.0 (446.96216216216214) + | > loss_1: 30.79405975341797 (30.657188982576937) + | > grad_norm_1: tensor(175.4372, device='cuda:0') (tensor(208.7187, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.8601 (0.8287827375772837) + | > loader_time: 0.0122 (0.01086073823877284) + + + --> TIME: 2024-06-25 09:22:09 -- STEP: 210/591 -- GLOBAL_STEP: 23850 + | > loss_disc: 2.66325044631958 (2.7079487266994646) + | > loss_disc_real_0: 0.18984869122505188 (0.2050846350689729) + | > loss_disc_real_1: 0.24385038018226624 (0.23033194754804884) + | > loss_disc_real_2: 0.2244935929775238 (0.2354545921087265) + | > loss_disc_real_3: 0.2635035216808319 (0.2358838372287296) + | > loss_disc_real_4: 0.20670752227306366 (0.2319555484822818) + | > loss_disc_real_5: 0.21306759119033813 (0.23835484761567344) + | > loss_0: 2.66325044631958 (2.7079487266994646) + | > grad_norm_0: tensor(3.6390, device='cuda:0') (tensor(11.1772, device='cuda:0')) + | > loss_gen: 2.040073871612549 (1.958363057885851) + | > loss_kl: 2.4736554622650146 (2.5350033862250183) + | > loss_feat: 4.100098133087158 (3.959352231025696) + | > loss_mel: 20.89208221435547 (20.840315128508056) + | > loss_duration: 1.3359146118164062 (1.3449625026612055) + | > amp_scaler: 256.0 (424.22857142857146) + | > loss_1: 30.84182357788086 (30.63799637385777) + | > grad_norm_1: tensor(53.5710, device='cuda:0') (tensor(196.9670, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.8782 (0.8343012809753418) + | > loader_time: 0.0127 (0.011135847227914001) + + + --> TIME: 2024-06-25 09:22:32 -- STEP: 235/591 -- GLOBAL_STEP: 23875 + | > loss_disc: 2.6707615852355957 (2.70580195366068) + | > loss_disc_real_0: 0.13967445492744446 (0.2044835660368838) + | > loss_disc_real_1: 0.21605387330055237 (0.2293571945200575) + | > loss_disc_real_2: 0.24861213564872742 (0.23561532643247157) + | > loss_disc_real_3: 0.24145501852035522 (0.23584000943823064) + | > loss_disc_real_4: 0.24079002439975739 (0.23249299697419432) + | > loss_disc_real_5: 0.23606212437152863 (0.23856915799861259) + | > loss_0: 2.6707615852355957 (2.70580195366068) + | > grad_norm_0: tensor(16.6251, device='cuda:0') (tensor(10.9181, device='cuda:0')) + | > loss_gen: 2.0400233268737793 (1.9550200893523846) + | > loss_kl: 2.43754243850708 (2.5285067091596876) + | > loss_feat: 3.9276363849639893 (3.9473126127364786) + | > loss_mel: 20.323944091796875 (20.822858924054074) + | > loss_duration: 1.3227638006210327 (1.3449469992455017) + | > amp_scaler: 256.0 (406.331914893617) + | > loss_1: 30.051910400390625 (30.59864536853547) + | > grad_norm_1: tensor(53.2198, device='cuda:0') (tensor(190.8554, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.8815 (0.8393557152849562) + | > loader_time: 0.0127 (0.011483768706626088) + + + --> TIME: 2024-06-25 09:22:55 -- STEP: 260/591 -- GLOBAL_STEP: 23900 + | > loss_disc: 2.6639575958251953 (2.7061806101065407) + | > loss_disc_real_0: 0.19798579812049866 (0.20478412583470346) + | > loss_disc_real_1: 0.21374359726905823 (0.22920222603357754) + | > loss_disc_real_2: 0.22315534949302673 (0.23569371849298476) + | > loss_disc_real_3: 0.22767280042171478 (0.23537868857383729) + | > loss_disc_real_4: 0.24120232462882996 (0.23258430413328685) + | > loss_disc_real_5: 0.2455645054578781 (0.2387381893511002) + | > loss_0: 2.6639575958251953 (2.7061806101065407) + | > grad_norm_0: tensor(4.0682, device='cuda:0') (tensor(10.9284, device='cuda:0')) + | > loss_gen: 1.9411696195602417 (1.951530811878351) + | > loss_kl: 2.5002057552337646 (2.5242573343790484) + | > loss_feat: 4.011470317840576 (3.943389326792497) + | > loss_mel: 20.614925384521484 (20.81164889702429) + | > loss_duration: 1.3296678066253662 (1.3450255591135758) + | > amp_scaler: 256.0 (391.87692307692305) + | > loss_1: 30.39743995666504 (30.575851990626408) + | > grad_norm_1: tensor(148.0733, device='cuda:0') (tensor(181.8082, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.8921 (0.8452938850109394) + | > loader_time: 0.0126 (0.011768629000737123) + + + --> TIME: 2024-06-25 09:23:18 -- STEP: 285/591 -- GLOBAL_STEP: 23925 + | > loss_disc: 2.6868748664855957 (2.705463731498049) + | > loss_disc_real_0: 0.21091026067733765 (0.20513525189537757) + | > loss_disc_real_1: 0.23912008106708527 (0.22870486389126693) + | > loss_disc_real_2: 0.23773454129695892 (0.23544107555297383) + | > loss_disc_real_3: 0.3060145676136017 (0.23543076081234113) + | > loss_disc_real_4: 0.23232707381248474 (0.23247950500563572) + | > loss_disc_real_5: 0.22807800769805908 (0.2387370114786583) + | > loss_0: 2.6868748664855957 (2.705463731498049) + | > grad_norm_0: tensor(6.9401, device='cuda:0') (tensor(11.0625, device='cuda:0')) + | > loss_gen: 1.819581389427185 (1.9499931230879666) + | > loss_kl: 2.517322301864624 (2.5219797000550375) + | > loss_feat: 3.7368526458740234 (3.9400902957247013) + | > loss_mel: 20.403722763061523 (20.80982686762223) + | > loss_duration: 1.345738172531128 (1.345728724044666) + | > amp_scaler: 256.0 (379.9578947368421) + | > loss_1: 29.823217391967773 (30.567618754872104) + | > grad_norm_1: tensor(286.5503, device='cuda:0') (tensor(174.8412, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.9257 (0.8505988823740105) + | > loader_time: 0.0154 (0.012021815149407646) + + + --> TIME: 2024-06-25 09:23:42 -- STEP: 310/591 -- GLOBAL_STEP: 23950 + | > loss_disc: 2.7235374450683594 (2.7065470956986957) + | > loss_disc_real_0: 0.20926347374916077 (0.20499658562964013) + | > loss_disc_real_1: 0.2684817910194397 (0.2286049684689891) + | > loss_disc_real_2: 0.248396098613739 (0.23578812268472488) + | > loss_disc_real_3: 0.2737768292427063 (0.23570411344689707) + | > loss_disc_real_4: 0.26443642377853394 (0.23267679661512375) + | > loss_disc_real_5: 0.2767038345336914 (0.23889717526974216) + | > loss_0: 2.7235374450683594 (2.7065470956986957) + | > grad_norm_0: tensor(6.3033, device='cuda:0') (tensor(10.9729, device='cuda:0')) + | > loss_gen: 1.979520320892334 (1.9488956985935089) + | > loss_kl: 2.5930683612823486 (2.52270192946157) + | > loss_feat: 3.826303720474243 (3.932516505641322) + | > loss_mel: 20.609989166259766 (20.818100190931744) + | > loss_duration: 1.3651165962219238 (1.34646330841126) + | > amp_scaler: 256.0 (369.9612903225806) + | > loss_1: 30.373998641967773 (30.56867764995944) + | > grad_norm_1: tensor(56.6545, device='cuda:0') (tensor(170.4654, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.9338 (0.8564657334358462) + | > loader_time: 0.0158 (0.01231716986625426) + + + --> TIME: 2024-06-25 09:24:05 -- STEP: 335/591 -- GLOBAL_STEP: 23975 + | > loss_disc: 2.7373857498168945 (2.7055931262115953) + | > loss_disc_real_0: 0.196104496717453 (0.20498717642986958) + | > loss_disc_real_1: 0.2187877744436264 (0.2281583914561058) + | > loss_disc_real_2: 0.23097841441631317 (0.23568430840969085) + | > loss_disc_real_3: 0.24728623032569885 (0.2354303430265455) + | > loss_disc_real_4: 0.23820239305496216 (0.23279871073231767) + | > loss_disc_real_5: 0.24560528993606567 (0.23909676822263803) + | > loss_0: 2.7373857498168945 (2.7055931262115953) + | > grad_norm_0: tensor(6.0227, device='cuda:0') (tensor(10.8416, device='cuda:0')) + | > loss_gen: 1.8033350706100464 (1.9461421023553878) + | > loss_kl: 2.5297648906707764 (2.5209422517178637) + | > loss_feat: 3.7362234592437744 (3.92460406502681) + | > loss_mel: 20.925451278686523 (20.81597674924935) + | > loss_duration: 1.3698623180389404 (1.347292813258385) + | > amp_scaler: 256.0 (361.45671641791046) + | > loss_1: 30.36463737487793 (30.554957990504022) + | > grad_norm_1: tensor(110.8965, device='cuda:0') (tensor(170.5524, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.9416 (0.8620023713183047) + | > loader_time: 0.0164 (0.012575185832692624) + + + --> TIME: 2024-06-25 09:24:30 -- STEP: 360/591 -- GLOBAL_STEP: 24000 + | > loss_disc: 2.7355194091796875 (2.7038375337918614) + | > loss_disc_real_0: 0.22898158431053162 (0.20477819231649244) + | > loss_disc_real_1: 0.22327326238155365 (0.22778268332282703) + | > loss_disc_real_2: 0.2180812954902649 (0.23542800976170433) + | > loss_disc_real_3: 0.27679985761642456 (0.23527356787688203) + | > loss_disc_real_4: 0.22224341332912445 (0.23252609454923207) + | > loss_disc_real_5: 0.2279040813446045 (0.2389935656140248) + | > loss_0: 2.7355194091796875 (2.7038375337918614) + | > grad_norm_0: tensor(6.5439, device='cuda:0') (tensor(10.8609, device='cuda:0')) + | > loss_gen: 1.9013350009918213 (1.9458405951658884) + | > loss_kl: 2.3697643280029297 (2.5184177080790184) + | > loss_feat: 3.8423349857330322 (3.9194760077529485) + | > loss_mel: 20.665651321411133 (20.805740727318646) + | > loss_duration: 1.3531028032302856 (1.348719442221854) + | > amp_scaler: 256.0 (354.1333333333334) + | > loss_1: 30.13218879699707 (30.53819449742635) + | > grad_norm_1: tensor(58.5194, device='cuda:0') (tensor(175.8937, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.9731 (0.8677839285797543) + | > loader_time: 0.0175 (0.012854264179865526) + + + --> TIME: 2024-06-25 09:24:54 -- STEP: 385/591 -- GLOBAL_STEP: 24025 + | > loss_disc: 2.6766881942749023 (2.7045793527132522) + | > loss_disc_real_0: 0.14340195059776306 (0.20508675981651656) + | > loss_disc_real_1: 0.2293836623430252 (0.22775758199103466) + | > loss_disc_real_2: 0.2351912558078766 (0.23551783062420883) + | > loss_disc_real_3: 0.19982948899269104 (0.2351397083177195) + | > loss_disc_real_4: 0.22370973229408264 (0.2324996565456514) + | > loss_disc_real_5: 0.21158307790756226 (0.23891052363754867) + | > loss_0: 2.6766881942749023 (2.7045793527132522) + | > grad_norm_0: tensor(16.0341, device='cuda:0') (tensor(10.9210, device='cuda:0')) + | > loss_gen: 1.9781296253204346 (1.9439961238340897) + | > loss_kl: 2.5502662658691406 (2.5165258444748897) + | > loss_feat: 3.8612232208251953 (3.9083206083867457) + | > loss_mel: 20.893484115600586 (20.795316785341715) + | > loss_duration: 1.3552294969558716 (1.3500166168460601) + | > amp_scaler: 256.0 (347.7610389610391) + | > loss_1: 30.638334274291992 (30.514176004583184) + | > grad_norm_1: tensor(161.6307, device='cuda:0') (tensor(177.7882, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.9513 (0.8738842091003022) + | > loader_time: 0.0175 (0.013105573901882427) + + + --> TIME: 2024-06-25 09:25:19 -- STEP: 410/591 -- GLOBAL_STEP: 24050 + | > loss_disc: 2.7205238342285156 (2.706103605177346) + | > loss_disc_real_0: 0.17644831538200378 (0.2054741219776433) + | > loss_disc_real_1: 0.224220409989357 (0.227638327711966) + | > loss_disc_real_2: 0.24568480253219604 (0.23581952954210886) + | > loss_disc_real_3: 0.21802012622356415 (0.23529111208712183) + | > loss_disc_real_4: 0.22977322340011597 (0.23273292707233895) + | > loss_disc_real_5: 0.22973205149173737 (0.23914570441333258) + | > loss_0: 2.7205238342285156 (2.706103605177346) + | > grad_norm_0: tensor(5.5444, device='cuda:0') (tensor(11.0422, device='cuda:0')) + | > loss_gen: 1.7168867588043213 (1.9419723874185144) + | > loss_kl: 2.4714603424072266 (2.515295510757258) + | > loss_feat: 3.495812177658081 (3.8967559599294894) + | > loss_mel: 20.507022857666016 (20.784472846984844) + | > loss_duration: 1.3811407089233398 (1.3507336639776466) + | > amp_scaler: 256.0 (342.1658536585367) + | > loss_1: 29.572322845458984 (30.489230411808666) + | > grad_norm_1: tensor(33.2077, device='cuda:0') (tensor(175.3701, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 0.9679 (0.8801289319992065) + | > loader_time: 0.0191 (0.013335965319377627) + + + --> TIME: 2024-06-25 09:25:45 -- STEP: 435/591 -- GLOBAL_STEP: 24075 + | > loss_disc: 2.8054721355438232 (2.706598531788795) + | > loss_disc_real_0: 0.1621461659669876 (0.20551679781798662) + | > loss_disc_real_1: 0.21446120738983154 (0.22749328692068999) + | > loss_disc_real_2: 0.2374817430973053 (0.23571007618273812) + | > loss_disc_real_3: 0.2512863278388977 (0.2352025403373543) + | > loss_disc_real_4: 0.2579211890697479 (0.23285205158694036) + | > loss_disc_real_5: 0.2612189054489136 (0.23913621748315878) + | > loss_0: 2.8054721355438232 (2.706598531788795) + | > grad_norm_0: tensor(9.4895, device='cuda:0') (tensor(11.0448, device='cuda:0')) + | > loss_gen: 1.9572365283966064 (1.9401885106645782) + | > loss_kl: 2.325453281402588 (2.5122151056925435) + | > loss_feat: 3.393238067626953 (3.885928598491625) + | > loss_mel: 19.786422729492188 (20.768183004445028) + | > loss_duration: 1.3745896816253662 (1.351678470633496) + | > amp_scaler: 256.0 (337.2137931034484) + | > loss_1: 28.83694076538086 (30.458193706643993) + | > grad_norm_1: tensor(79.0434, device='cuda:0') (tensor(170.2956, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 1.0028 (0.886748413107861) + | > loader_time: 0.0201 (0.013595835915927236) + + + --> TIME: 2024-06-25 09:26:10 -- STEP: 460/591 -- GLOBAL_STEP: 24100 + | > loss_disc: 2.7624380588531494 (2.7065953679706753) + | > loss_disc_real_0: 0.2197793424129486 (0.20544657344403477) + | > loss_disc_real_1: 0.26527097821235657 (0.22747505187340405) + | > loss_disc_real_2: 0.2538745701313019 (0.23576712546789128) + | > loss_disc_real_3: 0.22332362830638885 (0.23516808476137077) + | > loss_disc_real_4: 0.27090829610824585 (0.23318388727696046) + | > loss_disc_real_5: 0.26228201389312744 (0.23921536618600722) + | > loss_0: 2.7624380588531494 (2.7065953679706753) + | > grad_norm_0: tensor(6.5146, device='cuda:0') (tensor(10.9383, device='cuda:0')) + | > loss_gen: 1.773075819015503 (1.9381690616193026) + | > loss_kl: 2.4230432510375977 (2.511210945896478) + | > loss_feat: 3.5625245571136475 (3.876650527249212) + | > loss_mel: 20.50817108154297 (20.752121096071974) + | > loss_duration: 1.3837146759033203 (1.3526765937390537) + | > amp_scaler: 256.0 (332.8000000000001) + | > loss_1: 29.650529861450195 (30.430828243753183) + | > grad_norm_1: tensor(143.1015, device='cuda:0') (tensor(170.0282, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 1.0096 (0.8931984129159347) + | > loader_time: 0.0214 (0.013854483936143963) + + + --> TIME: 2024-06-25 09:26:36 -- STEP: 485/591 -- GLOBAL_STEP: 24125 + | > loss_disc: 2.66937255859375 (2.7068430753098336) + | > loss_disc_real_0: 0.17395566403865814 (0.2052845829848162) + | > loss_disc_real_1: 0.20763880014419556 (0.2273547088976988) + | > loss_disc_real_2: 0.24410757422447205 (0.23570191946226296) + | > loss_disc_real_3: 0.19950023293495178 (0.23528527847884856) + | > loss_disc_real_4: 0.23792698979377747 (0.23312742258470082) + | > loss_disc_real_5: 0.2484925240278244 (0.23924024999141694) + | > loss_0: 2.66937255859375 (2.7068430753098336) + | > grad_norm_0: tensor(6.7812, device='cuda:0') (tensor(10.8361, device='cuda:0')) + | > loss_gen: 1.9188318252563477 (1.9367267387429463) + | > loss_kl: 2.4851832389831543 (2.509249809107828) + | > loss_feat: 3.6293513774871826 (3.8653915100490925) + | > loss_mel: 20.27143096923828 (20.732646835956363) + | > loss_duration: 1.3812127113342285 (1.3540239599562185) + | > amp_scaler: 256.0 (328.8412371134022) + | > loss_1: 29.686010360717773 (30.398038864135742) + | > grad_norm_1: tensor(232.5551, device='cuda:0') (tensor(173.5307, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 1.0224 (0.9000278841588915) + | > loader_time: 0.0185 (0.014119745529804039) + + + --> TIME: 2024-06-25 09:27:04 -- STEP: 510/591 -- GLOBAL_STEP: 24150 + | > loss_disc: 2.7265000343322754 (2.7075598777509216) + | > loss_disc_real_0: 0.25541332364082336 (0.2055746581770626) + | > loss_disc_real_1: 0.19749578833580017 (0.2273462870541741) + | > loss_disc_real_2: 0.21011056005954742 (0.23571399222402012) + | > loss_disc_real_3: 0.2536146640777588 (0.23535357307569654) + | > loss_disc_real_4: 0.2547878921031952 (0.23297940784809637) + | > loss_disc_real_5: 0.24797874689102173 (0.23923386960637336) + | > loss_0: 2.7265000343322754 (2.7075598777509216) + | > grad_norm_0: tensor(15.6856, device='cuda:0') (tensor(11.0043, device='cuda:0')) + | > loss_gen: 1.786484956741333 (1.9358904530020322) + | > loss_kl: 2.4291396141052246 (2.5067851314357665) + | > loss_feat: 3.535811185836792 (3.856513398768855) + | > loss_mel: 20.27031898498535 (20.72156720254934) + | > loss_duration: 1.3868122100830078 (1.3556203517259338) + | > amp_scaler: 256.0 (325.2705882352942) + | > loss_1: 29.408567428588867 (30.3763765372482) + | > grad_norm_1: tensor(256.0014, device='cuda:0') (tensor(176.9604, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 1.0619 (0.908885347141939) + | > loader_time: 0.017 (0.014368159163231949) + + + --> TIME: 2024-06-25 09:27:33 -- STEP: 535/591 -- GLOBAL_STEP: 24175 + | > loss_disc: 2.6740646362304688 (2.7073040119955487) + | > loss_disc_real_0: 0.21244503557682037 (0.20567080485765069) + | > loss_disc_real_1: 0.24105118215084076 (0.22740440156972297) + | > loss_disc_real_2: 0.2289721816778183 (0.23584375921810902) + | > loss_disc_real_3: 0.24714326858520508 (0.23544119599266586) + | > loss_disc_real_4: 0.26633769273757935 (0.2326104477744236) + | > loss_disc_real_5: 0.2376818209886551 (0.23909336049980096) + | > loss_0: 2.6740646362304688 (2.7073040119955487) + | > grad_norm_0: tensor(8.1503, device='cuda:0') (tensor(11.1008, device='cuda:0')) + | > loss_gen: 1.8813860416412354 (1.9374311012642407) + | > loss_kl: 2.3768508434295654 (2.5042689015932154) + | > loss_feat: 3.7260780334472656 (3.8534484110145923) + | > loss_mel: 20.415557861328125 (20.714197825494203) + | > loss_duration: 1.3576412200927734 (1.3572029543814261) + | > amp_scaler: 256.0 (322.0336448598132) + | > loss_1: 29.75751304626465 (30.36654918349792) + | > grad_norm_1: tensor(78.6311, device='cuda:0') (tensor(183.9529, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 1.199 (0.9191945477066752) + | > loader_time: 0.0215 (0.0146608593307923) + + + --> TIME: 2024-06-25 09:28:03 -- STEP: 560/591 -- GLOBAL_STEP: 24200 + | > loss_disc: 2.7554283142089844 (2.706792308177268) + | > loss_disc_real_0: 0.24844014644622803 (0.20561572748369406) + | > loss_disc_real_1: 0.24923047423362732 (0.22723294094737082) + | > loss_disc_real_2: 0.2278921902179718 (0.23573663617883409) + | > loss_disc_real_3: 0.23810574412345886 (0.23528518572981869) + | > loss_disc_real_4: 0.2243836224079132 (0.23266281027879035) + | > loss_disc_real_5: 0.23813335597515106 (0.2391764008040939) + | > loss_0: 2.7554283142089844 (2.706792308177268) + | > grad_norm_0: tensor(7.6184, device='cuda:0') (tensor(10.9763, device='cuda:0')) + | > loss_gen: 1.930645227432251 (1.9368728439722742) + | > loss_kl: 2.4342033863067627 (2.503557284389222) + | > loss_feat: 3.360990285873413 (3.8456777074507307) + | > loss_mel: 20.331769943237305 (20.701510524749743) + | > loss_duration: 1.386150598526001 (1.3588179203016426) + | > amp_scaler: 256.0 (319.0857142857145) + | > loss_1: 29.44375991821289 (30.346436268942696) + | > grad_norm_1: tensor(268.9403, device='cuda:0') (tensor(182.4686, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 1.2946 (0.9311866044998169) + | > loader_time: 0.0288 (0.015013474225997931) + + + --> TIME: 2024-06-25 09:28:35 -- STEP: 585/591 -- GLOBAL_STEP: 24225 + | > loss_disc: 2.70393443107605 (2.706655889087255) + | > loss_disc_real_0: 0.238124281167984 (0.20554590118237032) + | > loss_disc_real_1: 0.2300713211297989 (0.2271921294367211) + | > loss_disc_real_2: 0.2566949427127838 (0.23565661053881687) + | > loss_disc_real_3: 0.25719398260116577 (0.23532043433087504) + | > loss_disc_real_4: 0.24331429600715637 (0.23264098131758534) + | > loss_disc_real_5: 0.2376822531223297 (0.23923091144643277) + | > loss_0: 2.70393443107605 (2.706655889087255) + | > grad_norm_0: tensor(8.5110, device='cuda:0') (tensor(10.8769, device='cuda:0')) + | > loss_gen: 1.8271398544311523 (1.9357004188064837) + | > loss_kl: 2.468873977661133 (2.5016105708912897) + | > loss_feat: 3.7505736351013184 (3.8370764504131087) + | > loss_mel: 20.015676498413086 (20.685891466059218) + | > loss_duration: 1.4082911014556885 (1.3605586991350886) + | > amp_scaler: 256.0 (316.3897435897437) + | > loss_1: 29.47055435180664 (30.320837597969252) + | > grad_norm_1: tensor(43.0754, device='cuda:0') (tensor(181.6664, device='cuda:0')) + | > current_lr_0: 0.00019900243364508313 + | > current_lr_1: 0.00019900243364508313 + | > step_time: 1.2704 (0.9441857097495315) + | > loader_time: 0.0213 (0.015359788470798075) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036825880091241065 (+2.5277442120491024e-05) + | > avg_loss_disc: 2.768124392692079 (+0.08706853237557954) + | > avg_loss_disc_real_0: 0.22061158019177457 (-0.0202396499349716) + | > avg_loss_disc_real_1: 0.2286184569622608 (+0.016525202292077074) + | > avg_loss_disc_real_2: 0.23024619228028237 (-0.011024675470717393) + | > avg_loss_disc_real_3: 0.24673478273635216 (+0.014782661136160502) + | > avg_loss_disc_real_4: 0.2223566139632083 (-0.02393170176668369) + | > avg_loss_disc_real_5: 0.2238455227714904 (-0.014975450774456583) + | > avg_loss_0: 2.768124392692079 (+0.08706853237557954) + | > avg_loss_gen: 1.8081698468390932 (-0.13226337128497168) + | > avg_loss_kl: 2.5854325903222914 (+0.012151089120418401) + | > avg_loss_feat: 3.7289791462269237 (+0.026957166955826395) + | > avg_loss_mel: 20.645329333366234 (+0.005985341173538217) + | > avg_loss_duration: 1.3877743203589257 (+0.011356092513875815) + | > avg_loss_1: 30.155685343640915 (-0.07581345578457643) + + + > EPOCH: 42/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 09:28:51)  + + --> TIME: 2024-06-25 09:29:07 -- STEP: 19/591 -- GLOBAL_STEP: 24250 + | > loss_disc: 2.6686065196990967 (2.7162746504733435) + | > loss_disc_real_0: 0.14061054587364197 (0.20807088793892609) + | > loss_disc_real_1: 0.20609833300113678 (0.22585248162871913) + | > loss_disc_real_2: 0.22218438982963562 (0.23648696039852343) + | > loss_disc_real_3: 0.20265284180641174 (0.23523396096731486) + | > loss_disc_real_4: 0.22325970232486725 (0.23723700328877098) + | > loss_disc_real_5: 0.2286929190158844 (0.24269720911979675) + | > loss_0: 2.6686065196990967 (2.7162746504733435) + | > grad_norm_0: tensor(15.4528, device='cuda:0') (tensor(12.0101, device='cuda:0')) + | > loss_gen: 1.9852291345596313 (1.944579513449418) + | > loss_kl: 2.538362979888916 (2.647347425159655) + | > loss_feat: 3.8178346157073975 (4.040058851242065) + | > loss_mel: 20.309185028076172 (20.690322474429482) + | > loss_duration: 1.3210468292236328 (1.3266963645031578) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.97165870666504 (30.649004685251338) + | > grad_norm_1: tensor(111.0782, device='cuda:0') (tensor(124.1302, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.7784 (0.7717067567925704) + | > loader_time: 0.0091 (0.008859960656417044) + + + --> TIME: 2024-06-25 09:29:27 -- STEP: 44/591 -- GLOBAL_STEP: 24275 + | > loss_disc: 2.6013832092285156 (2.707681872627951) + | > loss_disc_real_0: 0.14233911037445068 (0.20624366351826626) + | > loss_disc_real_1: 0.21585394442081451 (0.22449591654268178) + | > loss_disc_real_2: 0.20617759227752686 (0.23506435650316151) + | > loss_disc_real_3: 0.2096979171037674 (0.23539028282869945) + | > loss_disc_real_4: 0.22933630645275116 (0.2350061092187058) + | > loss_disc_real_5: 0.23091033101081848 (0.24147515202110464) + | > loss_0: 2.6013832092285156 (2.707681872627951) + | > grad_norm_0: tensor(17.3516, device='cuda:0') (tensor(10.1837, device='cuda:0')) + | > loss_gen: 2.141808271408081 (1.9330991506576538) + | > loss_kl: 2.5531656742095947 (2.620610621842471) + | > loss_feat: 4.095699310302734 (3.9703468192707394) + | > loss_mel: 20.803373336791992 (20.709871725602582) + | > loss_duration: 1.3387396335601807 (1.3299546891992742) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.932785034179688 (30.563882957805287) + | > grad_norm_1: tensor(120.0274, device='cuda:0') (tensor(116.4093, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.7981 (0.7863490852442655) + | > loader_time: 0.0107 (0.009447910568930885) + + + --> TIME: 2024-06-25 09:29:48 -- STEP: 69/591 -- GLOBAL_STEP: 24300 + | > loss_disc: 2.7613017559051514 (2.7026306131611695) + | > loss_disc_real_0: 0.1498742401599884 (0.20668678104445554) + | > loss_disc_real_1: 0.22624781727790833 (0.22459456886070361) + | > loss_disc_real_2: 0.20819807052612305 (0.23483405143454456) + | > loss_disc_real_3: 0.27467015385627747 (0.23599206364673117) + | > loss_disc_real_4: 0.23580117523670197 (0.2334997181011283) + | > loss_disc_real_5: 0.2472693920135498 (0.24017848765504532) + | > loss_0: 2.7613017559051514 (2.7026306131611695) + | > grad_norm_0: tensor(10.5513, device='cuda:0') (tensor(10.4224, device='cuda:0')) + | > loss_gen: 1.8873538970947266 (1.939198965611665) + | > loss_kl: 2.541731834411621 (2.584817457890165) + | > loss_feat: 4.0223774909973145 (3.975350960441257) + | > loss_mel: 20.760906219482422 (20.734872043996617) + | > loss_duration: 1.3398287296295166 (1.3375976534857263) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.55219841003418 (30.57183691384136) + | > grad_norm_1: tensor(411.7531, device='cuda:0') (tensor(179.4804, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.8162 (0.7962342759837275) + | > loader_time: 0.011 (0.009755037832951199) + + + --> TIME: 2024-06-25 09:30:09 -- STEP: 94/591 -- GLOBAL_STEP: 24325 + | > loss_disc: 2.7320525646209717 (2.7008165146442167) + | > loss_disc_real_0: 0.2294740229845047 (0.20554940854298306) + | > loss_disc_real_1: 0.30865752696990967 (0.22579070148949928) + | > loss_disc_real_2: 0.23849843442440033 (0.2352267419721218) + | > loss_disc_real_3: 0.283501535654068 (0.23509401606118424) + | > loss_disc_real_4: 0.2447821944952011 (0.23312796430384858) + | > loss_disc_real_5: 0.2408493310213089 (0.24025678444415965) + | > loss_0: 2.7320525646209717 (2.7008165146442167) + | > grad_norm_0: tensor(8.1977, device='cuda:0') (tensor(10.0989, device='cuda:0')) + | > loss_gen: 1.9881718158721924 (1.9447551077984748) + | > loss_kl: 2.524186849594116 (2.5714171161042887) + | > loss_feat: 4.025687217712402 (3.986115749846113) + | > loss_mel: 21.121809005737305 (20.801941587569857) + | > loss_duration: 1.3735930919647217 (1.3445458361443052) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.033449172973633 (30.648775344199322) + | > grad_norm_1: tensor(276.8605, device='cuda:0') (tensor(197.8038, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.8324 (0.8049887114382804) + | > loader_time: 0.0121 (0.010068287240697978) + + + --> TIME: 2024-06-25 09:30:31 -- STEP: 119/591 -- GLOBAL_STEP: 24350 + | > loss_disc: 2.7087764739990234 (2.7027866740186677) + | > loss_disc_real_0: 0.19850002229213715 (0.20618647180435037) + | > loss_disc_real_1: 0.28286099433898926 (0.2255523325014515) + | > loss_disc_real_2: 0.2336665391921997 (0.23531633552883854) + | > loss_disc_real_3: 0.22711127996444702 (0.23490675808000966) + | > loss_disc_real_4: 0.22981277108192444 (0.23290444672608576) + | > loss_disc_real_5: 0.2361694574356079 (0.240095899761224) + | > loss_0: 2.7087764739990234 (2.7027866740186677) + | > grad_norm_0: tensor(5.8042, device='cuda:0') (tensor(10.4776, device='cuda:0')) + | > loss_gen: 1.9504356384277344 (1.9425819641401787) + | > loss_kl: 2.535045623779297 (2.5569274385436254) + | > loss_feat: 4.091691493988037 (3.980155093329293) + | > loss_mel: 20.91114044189453 (20.837565269790773) + | > loss_duration: 1.33797287940979 (1.3459176646561182) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.82628631591797 (30.663147405416023) + | > grad_norm_1: tensor(336.6698, device='cuda:0') (tensor(215.7349, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.8538 (0.8120145717588794) + | > loader_time: 0.0122 (0.010360146770958135) + + + --> TIME: 2024-06-25 09:30:52 -- STEP: 144/591 -- GLOBAL_STEP: 24375 + | > loss_disc: 2.639240026473999 (2.701720029115677) + | > loss_disc_real_0: 0.2442859560251236 (0.20647040247503254) + | > loss_disc_real_1: 0.20078372955322266 (0.22530327033665445) + | > loss_disc_real_2: 0.2204694300889969 (0.23537946451041433) + | > loss_disc_real_3: 0.19347859919071198 (0.23471395247098473) + | > loss_disc_real_4: 0.22159893810749054 (0.23271831621726355) + | > loss_disc_real_5: 0.23828242719173431 (0.23976102409263453) + | > loss_0: 2.639240026473999 (2.701720029115677) + | > grad_norm_0: tensor(9.8479, device='cuda:0') (tensor(11.1522, device='cuda:0')) + | > loss_gen: 1.8824995756149292 (1.94260995172792) + | > loss_kl: 2.5063343048095703 (2.5493633631202917) + | > loss_feat: 3.967210531234741 (3.9779151711199017) + | > loss_mel: 20.84573745727539 (20.823078393936154) + | > loss_duration: 1.3336178064346313 (1.3455870043900278) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.535400390625 (30.63855387104882) + | > grad_norm_1: tensor(46.8434, device='cuda:0') (tensor(215.0894, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.8507 (0.8181578417619069) + | > loader_time: 0.0114 (0.010570958256721491) + + + --> TIME: 2024-06-25 09:31:14 -- STEP: 169/591 -- GLOBAL_STEP: 24400 + | > loss_disc: 2.7736892700195312 (2.7015221683231325) + | > loss_disc_real_0: 0.19034089148044586 (0.20577865041042925) + | > loss_disc_real_1: 0.22261115908622742 (0.22515736961153132) + | > loss_disc_real_2: 0.23826318979263306 (0.2355658430495911) + | > loss_disc_real_3: 0.23022760450839996 (0.23474859388974997) + | > loss_disc_real_4: 0.24216283857822418 (0.2327908601986586) + | > loss_disc_real_5: 0.23494304716587067 (0.23976512270918965) + | > loss_0: 2.7736892700195312 (2.7015221683231325) + | > grad_norm_0: tensor(6.6524, device='cuda:0') (tensor(10.9410, device='cuda:0')) + | > loss_gen: 1.989165186882019 (1.9411818741341313) + | > loss_kl: 2.600839614868164 (2.544101737660065) + | > loss_feat: 3.7914533615112305 (3.9710471559558393) + | > loss_mel: 20.06436538696289 (20.79392373350245) + | > loss_duration: 1.3331375122070312 (1.343957149770839) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.778959274291992 (30.5942116122274) + | > grad_norm_1: tensor(57.5303, device='cuda:0') (tensor(205.0858, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.8498 (0.8244470209765011) + | > loader_time: 0.0123 (0.010814769733586957) + + + --> TIME: 2024-06-25 09:31:36 -- STEP: 194/591 -- GLOBAL_STEP: 24425 + | > loss_disc: 2.7185544967651367 (2.701015504365115) + | > loss_disc_real_0: 0.15192098915576935 (0.20589719516868443) + | > loss_disc_real_1: 0.2229718267917633 (0.2253555047911467) + | > loss_disc_real_2: 0.23180927336215973 (0.23551801238785086) + | > loss_disc_real_3: 0.3221760392189026 (0.2348219415883428) + | > loss_disc_real_4: 0.23637346923351288 (0.23253078121192677) + | > loss_disc_real_5: 0.253372460603714 (0.2397131306148067) + | > loss_0: 2.7185544967651367 (2.701015504365115) + | > grad_norm_0: tensor(13.9137, device='cuda:0') (tensor(10.9110, device='cuda:0')) + | > loss_gen: 1.8534550666809082 (1.9400574847594978) + | > loss_kl: 2.5328521728515625 (2.539540934808476) + | > loss_feat: 3.9011666774749756 (3.967462712956458) + | > loss_mel: 20.862895965576172 (20.785674193470747) + | > loss_duration: 1.3498740196228027 (1.3431606083801124) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.500244140625 (30.575895938676656) + | > grad_norm_1: tensor(137.7673, device='cuda:0') (tensor(197.0797, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.8652 (0.8298812305804381) + | > loader_time: 0.0138 (0.011035744676884913) + + + --> TIME: 2024-06-25 09:31:59 -- STEP: 219/591 -- GLOBAL_STEP: 24450 + | > loss_disc: 2.6289243698120117 (2.7003211180369058) + | > loss_disc_real_0: 0.22901849448680878 (0.20590414198702328) + | > loss_disc_real_1: 0.24533002078533173 (0.22509311493401113) + | > loss_disc_real_2: 0.21631473302841187 (0.23543263760875893) + | > loss_disc_real_3: 0.2534373700618744 (0.23444979913430672) + | > loss_disc_real_4: 0.20785948634147644 (0.23245530272727688) + | > loss_disc_real_5: 0.19817893207073212 (0.23957539407629944) + | > loss_0: 2.6289243698120117 (2.7003211180369058) + | > grad_norm_0: tensor(10.4800, device='cuda:0') (tensor(10.8010, device='cuda:0')) + | > loss_gen: 1.9475427865982056 (1.9399185714112024) + | > loss_kl: 2.397759199142456 (2.533652074805133) + | > loss_feat: 4.113585472106934 (3.966581243358246) + | > loss_mel: 20.674287796020508 (20.790450866908245) + | > loss_duration: 1.3291372060775757 (1.3423533559389864) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.462312698364258 (30.572956154879915) + | > grad_norm_1: tensor(73.6282, device='cuda:0') (tensor(196.7919, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.8713 (0.8366405190942494) + | > loader_time: 0.0135 (0.011300169713965285) + + + --> TIME: 2024-06-25 09:32:22 -- STEP: 244/591 -- GLOBAL_STEP: 24475 + | > loss_disc: 2.769652843475342 (2.701680670019056) + | > loss_disc_real_0: 0.27806222438812256 (0.20573109055518127) + | > loss_disc_real_1: 0.21618542075157166 (0.22507249777678584) + | > loss_disc_real_2: 0.25677308440208435 (0.23582051297435996) + | > loss_disc_real_3: 0.25530385971069336 (0.23514170688195307) + | > loss_disc_real_4: 0.23628905415534973 (0.2328396374207051) + | > loss_disc_real_5: 0.2497028261423111 (0.23968480759468236) + | > loss_0: 2.769652843475342 (2.701680670019056) + | > grad_norm_0: tensor(11.2797, device='cuda:0') (tensor(10.8346, device='cuda:0')) + | > loss_gen: 1.8288724422454834 (1.9403620127771721) + | > loss_kl: 2.529536485671997 (2.533464209955247) + | > loss_feat: 3.817742347717285 (3.962513883582881) + | > loss_mel: 20.751636505126953 (20.788429627653027) + | > loss_duration: 1.3273983001708984 (1.3426238782093183) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.255186080932617 (30.56739363123159) + | > grad_norm_1: tensor(160.9810, device='cuda:0') (tensor(196.4511, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.9039 (0.8420539283361591) + | > loader_time: 0.0149 (0.011626060868872967) + + + --> TIME: 2024-06-25 09:32:45 -- STEP: 269/591 -- GLOBAL_STEP: 24500 + | > loss_disc: 2.768399238586426 (2.7009180965919914) + | > loss_disc_real_0: 0.302933931350708 (0.2058507060306666) + | > loss_disc_real_1: 0.2629062831401825 (0.22509016329249484) + | > loss_disc_real_2: 0.2387058436870575 (0.23562574220412727) + | > loss_disc_real_3: 0.23025186359882355 (0.23477746557347393) + | > loss_disc_real_4: 0.23296628892421722 (0.23256466618036248) + | > loss_disc_real_5: 0.20998887717723846 (0.23949857621166343) + | > loss_0: 2.768399238586426 (2.7009180965919914) + | > grad_norm_0: tensor(19.3764, device='cuda:0') (tensor(10.7442, device='cuda:0')) + | > loss_gen: 1.833580732345581 (1.9388322528853292) + | > loss_kl: 2.4615986347198486 (2.531863819710827) + | > loss_feat: 3.9419188499450684 (3.95623315134013) + | > loss_mel: 20.96630859375 (20.772018560246465) + | > loss_duration: 1.359441876411438 (1.3430258110996525) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.562849044799805 (30.541973610349746) + | > grad_norm_1: tensor(49.0485, device='cuda:0') (tensor(191.7893, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.8871 (0.8474017643130844) + | > loader_time: 0.0131 (0.01187058009179551) + + + --> TIME: 2024-06-25 09:33:08 -- STEP: 294/591 -- GLOBAL_STEP: 24525 + | > loss_disc: 2.72869873046875 (2.700803879166946) + | > loss_disc_real_0: 0.14626723527908325 (0.20537340696774373) + | > loss_disc_real_1: 0.20222119987010956 (0.22535724461484113) + | > loss_disc_real_2: 0.219455748796463 (0.2355256816562341) + | > loss_disc_real_3: 0.18446245789527893 (0.23467591430256968) + | > loss_disc_real_4: 0.2098558098077774 (0.2324922831386936) + | > loss_disc_real_5: 0.24729575216770172 (0.23960096931376426) + | > loss_0: 2.72869873046875 (2.700803879166946) + | > grad_norm_0: tensor(10.7063, device='cuda:0') (tensor(10.7497, device='cuda:0')) + | > loss_gen: 1.9474037885665894 (1.9385561667332032) + | > loss_kl: 2.5312705039978027 (2.5287205167368167) + | > loss_feat: 3.786184072494507 (3.9470579121388547) + | > loss_mel: 20.264253616333008 (20.749226349551666) + | > loss_duration: 1.3325269222259521 (1.3440685896646418) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.861637115478516 (30.507629504820116) + | > grad_norm_1: tensor(77.8510, device='cuda:0') (tensor(187.0315, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.9353 (0.8528347996627395) + | > loader_time: 0.0134 (0.012135113177656316) + + + --> TIME: 2024-06-25 09:33:31 -- STEP: 319/591 -- GLOBAL_STEP: 24550 + | > loss_disc: 2.663386344909668 (2.7017259224081496) + | > loss_disc_real_0: 0.262260377407074 (0.20515965168192096) + | > loss_disc_real_1: 0.23985035717487335 (0.22581956640680004) + | > loss_disc_real_2: 0.2368352711200714 (0.23581486221017509) + | > loss_disc_real_3: 0.18763715028762817 (0.23473967579091232) + | > loss_disc_real_4: 0.20500467717647552 (0.2325698308716747) + | > loss_disc_real_5: 0.2467934787273407 (0.23954146058768688) + | > loss_0: 2.663386344909668 (2.7017259224081496) + | > grad_norm_0: tensor(13.0846, device='cuda:0') (tensor(10.5327, device='cuda:0')) + | > loss_gen: 1.8262487649917603 (1.9375790779867141) + | > loss_kl: 2.5320053100585938 (2.527511520445535) + | > loss_feat: 3.8137245178222656 (3.9428967031939277) + | > loss_mel: 21.654024124145508 (20.76179111564421) + | > loss_duration: 1.3340904712677002 (1.3448569561620496) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.160093307495117 (30.514635331204694) + | > grad_norm_1: tensor(62.2615, device='cuda:0') (tensor(183.0141, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.9595 (0.8581873152323279) + | > loader_time: 0.0167 (0.012413432979284785) + + + --> TIME: 2024-06-25 09:33:55 -- STEP: 344/591 -- GLOBAL_STEP: 24575 + | > loss_disc: 2.690659761428833 (2.7017617184062344) + | > loss_disc_real_0: 0.1565198451280594 (0.20544448646521846) + | > loss_disc_real_1: 0.20190788805484772 (0.22566938447917612) + | > loss_disc_real_2: 0.24705485999584198 (0.23567327865680984) + | > loss_disc_real_3: 0.21964426338672638 (0.23473431100679) + | > loss_disc_real_4: 0.19758237898349762 (0.23249803128284077) + | > loss_disc_real_5: 0.22882786393165588 (0.2395408771239048) + | > loss_0: 2.690659761428833 (2.7017617184062344) + | > grad_norm_0: tensor(20.4993, device='cuda:0') (tensor(10.8507, device='cuda:0')) + | > loss_gen: 1.8296973705291748 (1.9375037481618482) + | > loss_kl: 2.5113143920898438 (2.5263651851997837) + | > loss_feat: 3.9708571434020996 (3.938953875802284) + | > loss_mel: 21.376754760742188 (20.760432819987454) + | > loss_duration: 1.35244882106781 (1.345981796467027) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.041072845458984 (30.509237383687218) + | > grad_norm_1: tensor(165.9519, device='cuda:0') (tensor(181.6643, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.9415 (0.8638960968616398) + | > loader_time: 0.0149 (0.012684255838394167) + + + --> TIME: 2024-06-25 09:34:20 -- STEP: 369/591 -- GLOBAL_STEP: 24600 + | > loss_disc: 2.6805996894836426 (2.70074578481638) + | > loss_disc_real_0: 0.19600354135036469 (0.20522762806758002) + | > loss_disc_real_1: 0.2501130998134613 (0.2255684233858656) + | > loss_disc_real_2: 0.22165854275226593 (0.23553191198081505) + | > loss_disc_real_3: 0.23149153590202332 (0.23476096709084704) + | > loss_disc_real_4: 0.2278819978237152 (0.2324263577458012) + | > loss_disc_real_5: 0.21651580929756165 (0.23937652754751323) + | > loss_0: 2.6805996894836426 (2.70074578481638) + | > grad_norm_0: tensor(4.8756, device='cuda:0') (tensor(10.8146, device='cuda:0')) + | > loss_gen: 1.999875545501709 (1.9376375452289736) + | > loss_kl: 2.4966585636138916 (2.525776180794571) + | > loss_feat: 3.7870187759399414 (3.9349214940213253) + | > loss_mel: 20.125211715698242 (20.753171047221038) + | > loss_duration: 1.3678340911865234 (1.3476195732752483) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.776599884033203 (30.499125777867427) + | > grad_norm_1: tensor(381.6458, device='cuda:0') (tensor(180.4832, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.9514 (0.8697993367668091) + | > loader_time: 0.0166 (0.012932712792704101) + + + --> TIME: 2024-06-25 09:34:44 -- STEP: 394/591 -- GLOBAL_STEP: 24625 + | > loss_disc: 2.6759235858917236 (2.6987639036275404) + | > loss_disc_real_0: 0.1484721153974533 (0.2047287210027881) + | > loss_disc_real_1: 0.21460960805416107 (0.22538119573429752) + | > loss_disc_real_2: 0.22246535122394562 (0.23539859427110799) + | > loss_disc_real_3: 0.24756769835948944 (0.23464854859609893) + | > loss_disc_real_4: 0.21996653079986572 (0.23238310255677566) + | > loss_disc_real_5: 0.23019391298294067 (0.2394391682547361) + | > loss_0: 2.6759235858917236 (2.6987639036275404) + | > grad_norm_0: tensor(9.3781, device='cuda:0') (tensor(10.6258, device='cuda:0')) + | > loss_gen: 1.9338151216506958 (1.9377161031447085) + | > loss_kl: 2.511641502380371 (2.5234038799547327) + | > loss_feat: 3.6770267486572266 (3.9328547794806776) + | > loss_mel: 19.65945816040039 (20.736985172717095) + | > loss_duration: 1.3342639207839966 (1.3483633456496418) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.1162052154541 (30.47932322255246) + | > grad_norm_1: tensor(185.6358, device='cuda:0') (tensor(179.7934, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.9565 (0.8757702668911311) + | > loader_time: 0.018 (0.013182587429956738) + + + --> TIME: 2024-06-25 09:35:09 -- STEP: 419/591 -- GLOBAL_STEP: 24650 + | > loss_disc: 2.6963462829589844 (2.6976764657332386) + | > loss_disc_real_0: 0.20273125171661377 (0.20485503943125785) + | > loss_disc_real_1: 0.21986432373523712 (0.2252945979625911) + | > loss_disc_real_2: 0.25189274549484253 (0.2352755251462818) + | > loss_disc_real_3: 0.24387845396995544 (0.23445176882647104) + | > loss_disc_real_4: 0.23927399516105652 (0.23219079561620454) + | > loss_disc_real_5: 0.22671258449554443 (0.23920457762152597) + | > loss_0: 2.6963462829589844 (2.6976764657332386) + | > grad_norm_0: tensor(7.2091, device='cuda:0') (tensor(10.6361, device='cuda:0')) + | > loss_gen: 2.00042986869812 (1.9382079939284587) + | > loss_kl: 2.4851126670837402 (2.522116464191522) + | > loss_feat: 3.837998628616333 (3.9295666200733415) + | > loss_mel: 21.265060424804688 (20.73396123280673) + | > loss_duration: 1.3780832290649414 (1.3494818443899224) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.966686248779297 (30.473334096212092) + | > grad_norm_1: tensor(350.8268, device='cuda:0') (tensor(180.9276, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 0.9598 (0.8818380229512947) + | > loader_time: 0.0188 (0.013467911035178103) + + + --> TIME: 2024-06-25 09:35:35 -- STEP: 444/591 -- GLOBAL_STEP: 24675 + | > loss_disc: 2.668994188308716 (2.6977297814042713) + | > loss_disc_real_0: 0.2087600827217102 (0.2049408191059892) + | > loss_disc_real_1: 0.22358985245227814 (0.22518402334671833) + | > loss_disc_real_2: 0.24442647397518158 (0.23539355256267497) + | > loss_disc_real_3: 0.22158671915531158 (0.23452561858806525) + | > loss_disc_real_4: 0.2221144586801529 (0.23213338650561668) + | > loss_disc_real_5: 0.2256152629852295 (0.23920828278537268) + | > loss_0: 2.668994188308716 (2.6977297814042713) + | > grad_norm_0: tensor(7.0040, device='cuda:0') (tensor(10.6978, device='cuda:0')) + | > loss_gen: 1.9746246337890625 (1.9381640725307636) + | > loss_kl: 2.4936821460723877 (2.521242423637496) + | > loss_feat: 4.12880277633667 (3.9238247849919774) + | > loss_mel: 20.175579071044922 (20.72769629847896) + | > loss_duration: 1.3707815408706665 (1.3504642403877534) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.143470764160156 (30.46139177545771) + | > grad_norm_1: tensor(107.0764, device='cuda:0') (tensor(182.6212, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 1.0216 (0.8884450433490514) + | > loader_time: 0.0165 (0.013755201756417217) + + + --> TIME: 2024-06-25 09:36:01 -- STEP: 469/591 -- GLOBAL_STEP: 24700 + | > loss_disc: 2.7243447303771973 (2.697984233339712) + | > loss_disc_real_0: 0.12486182153224945 (0.20504018788271622) + | > loss_disc_real_1: 0.22525738179683685 (0.2251148174947767) + | > loss_disc_real_2: 0.2577657699584961 (0.23540086411972289) + | > loss_disc_real_3: 0.2563959062099457 (0.2346254979560116) + | > loss_disc_real_4: 0.23275013267993927 (0.2321209475429836) + | > loss_disc_real_5: 0.26349228620529175 (0.23917180201265095) + | > loss_0: 2.7243447303771973 (2.697984233339712) + | > grad_norm_0: tensor(17.9745, device='cuda:0') (tensor(10.8337, device='cuda:0')) + | > loss_gen: 1.9873430728912354 (1.9376057294894382) + | > loss_kl: 2.4058029651641846 (2.5181739045612876) + | > loss_feat: 3.6684410572052 (3.914738967220412) + | > loss_mel: 20.386085510253906 (20.711939834074172) + | > loss_duration: 1.3716742992401123 (1.3516513678564954) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.819345474243164 (30.434109763041743) + | > grad_norm_1: tensor(47.2897, device='cuda:0') (tensor(179.7546, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 1.0058 (0.8955717478225481) + | > loader_time: 0.0155 (0.014052153650377352) + + + --> TIME: 2024-06-25 09:36:28 -- STEP: 494/591 -- GLOBAL_STEP: 24725 + | > loss_disc: 2.6698403358459473 (2.697828857522261) + | > loss_disc_real_0: 0.2760932743549347 (0.2050717176515081) + | > loss_disc_real_1: 0.21515721082687378 (0.22512451324023697) + | > loss_disc_real_2: 0.2515922784805298 (0.23525986012056288) + | > loss_disc_real_3: 0.2150142788887024 (0.23462603838458235) + | > loss_disc_real_4: 0.2250038981437683 (0.23225258863889253) + | > loss_disc_real_5: 0.21672362089157104 (0.2391156147607425) + | > loss_0: 2.6698403358459473 (2.697828857522261) + | > grad_norm_0: tensor(20.6371, device='cuda:0') (tensor(10.8125, device='cuda:0')) + | > loss_gen: 1.7879667282104492 (1.9368357392940445) + | > loss_kl: 2.3589818477630615 (2.515309012853185) + | > loss_feat: 3.739180564880371 (3.909346546238733) + | > loss_mel: 19.921960830688477 (20.696525075657647) + | > loss_duration: 1.3695672750473022 (1.3530201216940938) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.177658081054688 (30.41103644892272) + | > grad_norm_1: tensor(193.4290, device='cuda:0') (tensor(177.1925, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 1.0829 (0.903162042621659) + | > loader_time: 0.0231 (0.014378417358707322) + + + --> TIME: 2024-06-25 09:36:56 -- STEP: 519/591 -- GLOBAL_STEP: 24750 + | > loss_disc: 2.629657030105591 (2.6981504156410345) + | > loss_disc_real_0: 0.14154137670993805 (0.20485303866782859) + | > loss_disc_real_1: 0.18741698563098907 (0.2253578231169309) + | > loss_disc_real_2: 0.24466454982757568 (0.23536321713171024) + | > loss_disc_real_3: 0.26900845766067505 (0.23463702342528597) + | > loss_disc_real_4: 0.2160748690366745 (0.23217117637108745) + | > loss_disc_real_5: 0.23595786094665527 (0.2390792384198176) + | > loss_0: 2.629657030105591 (2.6981504156410345) + | > grad_norm_0: tensor(14.3053, device='cuda:0') (tensor(10.8549, device='cuda:0')) + | > loss_gen: 1.8608250617980957 (1.9365313817999956) + | > loss_kl: 2.444544553756714 (2.5135008066835227) + | > loss_feat: 4.065211296081543 (3.9037631816717013) + | > loss_mel: 21.078514099121094 (20.683858228326994) + | > loss_duration: 1.3732903003692627 (1.3547091431241045) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.822385787963867 (30.392362708530904) + | > grad_norm_1: tensor(407.4478, device='cuda:0') (tensor(179.8463, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 1.1776 (0.9130171980701661) + | > loader_time: 0.0232 (0.01469726966755009) + + + --> TIME: 2024-06-25 09:37:27 -- STEP: 544/591 -- GLOBAL_STEP: 24775 + | > loss_disc: 2.7029922008514404 (2.6986996544634594) + | > loss_disc_real_0: 0.169254332780838 (0.20495955754235826) + | > loss_disc_real_1: 0.27043038606643677 (0.22545029299662392) + | > loss_disc_real_2: 0.2456284761428833 (0.23538502798799207) + | > loss_disc_real_3: 0.2518605589866638 (0.23466430418193343) + | > loss_disc_real_4: 0.2393997609615326 (0.23205441984293215) + | > loss_disc_real_5: 0.22603069245815277 (0.23909038570983446) + | > loss_0: 2.7029922008514404 (2.6986996544634594) + | > grad_norm_0: tensor(6.4608, device='cuda:0') (tensor(10.9258, device='cuda:0')) + | > loss_gen: 1.9233373403549194 (1.9369657688281114) + | > loss_kl: 2.456024169921875 (2.511613292729156) + | > loss_feat: 3.844881057739258 (3.8969203325755455) + | > loss_mel: 20.50063705444336 (20.67112744906369) + | > loss_duration: 1.3887596130371094 (1.3562757307115725) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.11363983154297 (30.372902551118067) + | > grad_norm_1: tensor(351.1098, device='cuda:0') (tensor(185.9063, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 1.2991 (0.9258533492684367) + | > loader_time: 0.0239 (0.015042237937450412) + + + --> TIME: 2024-06-25 09:37:58 -- STEP: 569/591 -- GLOBAL_STEP: 24800 + | > loss_disc: 2.792562961578369 (2.698985150074078) + | > loss_disc_real_0: 0.35604020953178406 (0.20509388439213244) + | > loss_disc_real_1: 0.23820896446704865 (0.22540456920809815) + | > loss_disc_real_2: 0.22533829510211945 (0.23531354772604623) + | > loss_disc_real_3: 0.21842317283153534 (0.2346447691557068) + | > loss_disc_real_4: 0.2139875292778015 (0.2321568636357889) + | > loss_disc_real_5: 0.22632278501987457 (0.23914877630390477) + | > loss_0: 2.792562961578369 (2.698985150074078) + | > grad_norm_0: tensor(42.5530, device='cuda:0') (tensor(10.9624, device='cuda:0')) + | > loss_gen: 1.770117998123169 (1.9358321931114935) + | > loss_kl: 2.4555509090423584 (2.510204728332683) + | > loss_feat: 3.726308584213257 (3.8883181670009566) + | > loss_mel: 20.458871841430664 (20.64939398673381) + | > loss_duration: 1.3979270458221436 (1.35786121737978) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.80877685546875 (30.341610269932) + | > grad_norm_1: tensor(72.4062, device='cuda:0') (tensor(185.2902, device='cuda:0')) + | > current_lr_0: 0.0001989775583408775 + | > current_lr_1: 0.0001989775583408775 + | > step_time: 1.3081 (0.939746039613479) + | > loader_time: 0.0222 (0.015419906388267695) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00362522044080369 (-5.7367568320416343e-05) + | > avg_loss_disc: 2.788925931808796 (+0.020801539116717116) + | > avg_loss_disc_real_0: 0.394032450432473 (+0.1734208702406984) + | > avg_loss_disc_real_1: 0.2059208313201336 (-0.022697625642127184) + | > avg_loss_disc_real_2: 0.23509450256824493 (+0.004848310287962565) + | > avg_loss_disc_real_3: 0.24214449588288653 (-0.0045902868534656305) + | > avg_loss_disc_real_4: 0.23937540358685433 (+0.01701878962364603) + | > avg_loss_disc_real_5: 0.24334567499921678 (+0.019500152227726386) + | > avg_loss_0: 2.788925931808796 (+0.020801539116717116) + | > avg_loss_gen: 2.0998092027420703 (+0.29163935590297707) + | > avg_loss_kl: 2.546461333619788 (-0.03897125670250334) + | > avg_loss_feat: 3.716762380397066 (-0.012216765829857756) + | > avg_loss_mel: 20.338242064131066 (-0.3070872692351685) + | > avg_loss_duration: 1.3798108658892043 (-0.007963454469721354) + | > avg_loss_1: 30.08108605729773 (-0.07459928634318658) + + + > EPOCH: 43/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 09:38:35)  + + --> TIME: 2024-06-25 09:38:39 -- STEP: 3/591 -- GLOBAL_STEP: 24825 + | > loss_disc: 2.8441922664642334 (2.7547576427459717) + | > loss_disc_real_0: 0.4284709095954895 (0.22895280023415884) + | > loss_disc_real_1: 0.27861669659614563 (0.23555874824523926) + | > loss_disc_real_2: 0.25155913829803467 (0.2445673147837321) + | > loss_disc_real_3: 0.24185974895954132 (0.23936183750629425) + | > loss_disc_real_4: 0.2699072062969208 (0.25156549115975696) + | > loss_disc_real_5: 0.2900698184967041 (0.25876565277576447) + | > loss_0: 2.8441922664642334 (2.7547576427459717) + | > grad_norm_0: tensor(27.6196, device='cuda:0') (tensor(18.9911, device='cuda:0')) + | > loss_gen: 1.8118226528167725 (2.026940107345581) + | > loss_kl: 2.8433213233947754 (2.693901856740316) + | > loss_feat: 3.8649725914001465 (4.102038542429606) + | > loss_mel: 20.700008392333984 (20.907626469930012) + | > loss_duration: 1.3365895748138428 (1.3429102897644043) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.55671501159668 (31.073416392008465) + | > grad_norm_1: tensor(47.9122, device='cuda:0') (tensor(210.7114, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.7723 (0.7708536783854166) + | > loader_time: 0.008 (0.007622321446736653) + + + --> TIME: 2024-06-25 09:38:59 -- STEP: 28/591 -- GLOBAL_STEP: 24850 + | > loss_disc: 2.7706782817840576 (2.7145243201936995) + | > loss_disc_real_0: 0.3335005044937134 (0.20751232174890383) + | > loss_disc_real_1: 0.25466033816337585 (0.22781818945493018) + | > loss_disc_real_2: 0.25730466842651367 (0.23509512469172478) + | > loss_disc_real_3: 0.3001140058040619 (0.23878056609204837) + | > loss_disc_real_4: 0.234972283244133 (0.23592373090130941) + | > loss_disc_real_5: 0.22550763189792633 (0.2408064247242042) + | > loss_0: 2.7706782817840576 (2.7145243201936995) + | > grad_norm_0: tensor(25.0357, device='cuda:0') (tensor(11.4380, device='cuda:0')) + | > loss_gen: 1.8294545412063599 (1.9621839395591192) + | > loss_kl: 2.773240804672241 (2.669087367398398) + | > loss_feat: 3.7179715633392334 (4.0538917524474005) + | > loss_mel: 20.17559051513672 (20.674114772251674) + | > loss_duration: 1.3362023830413818 (1.3205910239900862) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.832460403442383 (30.679868902478898) + | > grad_norm_1: tensor(113.6582, device='cuda:0') (tensor(218.2142, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.7901 (0.7926844188145229) + | > loader_time: 0.009 (0.00885204757962908) + + + --> TIME: 2024-06-25 09:39:19 -- STEP: 53/591 -- GLOBAL_STEP: 24875 + | > loss_disc: 2.7593023777008057 (2.6929926917238056) + | > loss_disc_real_0: 0.22802424430847168 (0.2024864990193889) + | > loss_disc_real_1: 0.3066585063934326 (0.22658894450034736) + | > loss_disc_real_2: 0.26472723484039307 (0.23386752211822653) + | > loss_disc_real_3: 0.31272435188293457 (0.2361125664890937) + | > loss_disc_real_4: 0.25503337383270264 (0.23328242718048817) + | > loss_disc_real_5: 0.2682686746120453 (0.23919327978817922) + | > loss_0: 2.7593023777008057 (2.6929926917238056) + | > grad_norm_0: tensor(8.9849, device='cuda:0') (tensor(9.5605, device='cuda:0')) + | > loss_gen: 1.9335519075393677 (1.9605857898604195) + | > loss_kl: 2.4926674365997314 (2.634962054918397) + | > loss_feat: 3.734071969985962 (4.049768281432818) + | > loss_mel: 20.533933639526367 (20.67353248596192) + | > loss_duration: 1.3354110717773438 (1.3334962916824054) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.02963638305664 (30.65234490160672) + | > grad_norm_1: tensor(399.0703, device='cuda:0') (tensor(252.9746, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.8147 (0.7991783618927001) + | > loader_time: 0.0112 (0.009396355107145487) + + + --> TIME: 2024-06-25 09:39:40 -- STEP: 78/591 -- GLOBAL_STEP: 24900 + | > loss_disc: 2.6666393280029297 (2.6968992612300773) + | > loss_disc_real_0: 0.1628578156232834 (0.20308303240782175) + | > loss_disc_real_1: 0.19488346576690674 (0.22572294718179947) + | > loss_disc_real_2: 0.23751094937324524 (0.2354781163426546) + | > loss_disc_real_3: 0.22137345373630524 (0.23502783744763106) + | > loss_disc_real_4: 0.22163313627243042 (0.23372687437595466) + | > loss_disc_real_5: 0.22583962976932526 (0.23928836503854164) + | > loss_0: 2.6666393280029297 (2.6968992612300773) + | > grad_norm_0: tensor(10.7055, device='cuda:0') (tensor(10.1239, device='cuda:0')) + | > loss_gen: 2.04004168510437 (1.950996146752284) + | > loss_kl: 2.3932855129241943 (2.5980890408540382) + | > loss_feat: 3.761085271835327 (4.010408499302008) + | > loss_mel: 20.63671112060547 (20.68993270091522) + | > loss_duration: 1.3777098655700684 (1.340264827777178) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.208833694458008 (30.58969121101575) + | > grad_norm_1: tensor(53.8013, device='cuda:0') (tensor(233.8420, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.8335 (0.8060015867917966) + | > loader_time: 0.0114 (0.009702199544662081) + + + --> TIME: 2024-06-25 09:40:01 -- STEP: 103/591 -- GLOBAL_STEP: 24925 + | > loss_disc: 2.695277452468872 (2.698778541342726) + | > loss_disc_real_0: 0.1560077965259552 (0.20349982373633432) + | > loss_disc_real_1: 0.24145343899726868 (0.22592866362877262) + | > loss_disc_real_2: 0.2398039996623993 (0.23553927763573174) + | > loss_disc_real_3: 0.25536882877349854 (0.23465915183419161) + | > loss_disc_real_4: 0.24381399154663086 (0.23402225956754777) + | > loss_disc_real_5: 0.24064897000789642 (0.239738814749764) + | > loss_0: 2.695277452468872 (2.698778541342726) + | > grad_norm_0: tensor(15.5886, device='cuda:0') (tensor(10.6632, device='cuda:0')) + | > loss_gen: 2.013195753097534 (1.950125578537728) + | > loss_kl: 2.521080255508423 (2.5782067150745576) + | > loss_feat: 3.967350482940674 (4.004514509034387) + | > loss_mel: 20.49982261657715 (20.67892337539821) + | > loss_duration: 1.3328046798706055 (1.3456683101005924) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.33425521850586 (30.557438415231058) + | > grad_norm_1: tensor(279.6709, device='cuda:0') (tensor(228.3879, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.8401 (0.8123642356650344) + | > loader_time: 0.0109 (0.010035875931526847) + + + --> TIME: 2024-06-25 09:40:23 -- STEP: 128/591 -- GLOBAL_STEP: 24950 + | > loss_disc: 2.608792781829834 (2.700271189212799) + | > loss_disc_real_0: 0.12217994034290314 (0.20370610494865105) + | > loss_disc_real_1: 0.20397324860095978 (0.22614172648172826) + | > loss_disc_real_2: 0.22403396666049957 (0.2355015262728557) + | > loss_disc_real_3: 0.18972113728523254 (0.23465238546486944) + | > loss_disc_real_4: 0.22357165813446045 (0.23353021021466702) + | > loss_disc_real_5: 0.21793784201145172 (0.23919573705643415) + | > loss_0: 2.608792781829834 (2.700271189212799) + | > grad_norm_0: tensor(13.3147, device='cuda:0') (tensor(10.4732, device='cuda:0')) + | > loss_gen: 2.0711522102355957 (1.945501508191228) + | > loss_kl: 2.462611675262451 (2.56837011128664) + | > loss_feat: 4.073754787445068 (3.9927018433809276) + | > loss_mel: 20.1291561126709 (20.679645657539368) + | > loss_duration: 1.334444522857666 (1.3462166674435139) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.07111930847168 (30.532435670495033) + | > grad_norm_1: tensor(93.9140, device='cuda:0') (tensor(238.2964, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.8537 (0.8182852417230606) + | > loader_time: 0.0128 (0.010368699207901951) + + + --> TIME: 2024-06-25 09:40:45 -- STEP: 153/591 -- GLOBAL_STEP: 24975 + | > loss_disc: 2.721754550933838 (2.701411145964479) + | > loss_disc_real_0: 0.16713950037956238 (0.204647776348139) + | > loss_disc_real_1: 0.20352450013160706 (0.22559516560408024) + | > loss_disc_real_2: 0.24370096623897552 (0.23558388806246464) + | > loss_disc_real_3: 0.21861104667186737 (0.23538906648268107) + | > loss_disc_real_4: 0.2467600256204605 (0.23339846126394334) + | > loss_disc_real_5: 0.26539063453674316 (0.23949066612845152) + | > loss_0: 2.721754550933838 (2.701411145964479) + | > grad_norm_0: tensor(14.4669, device='cuda:0') (tensor(10.9077, device='cuda:0')) + | > loss_gen: 1.983065128326416 (1.9471199824139962) + | > loss_kl: 2.491081953048706 (2.5595049328274198) + | > loss_feat: 3.5217037200927734 (3.9896370881523184) + | > loss_mel: 20.326576232910156 (20.670436647203232) + | > loss_duration: 1.3157825469970703 (1.3444280943839373) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.63821029663086 (30.511126649145986) + | > grad_norm_1: tensor(168.1780, device='cuda:0') (tensor(228.8416, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.8605 (0.8236510831546159) + | > loader_time: 0.0122 (0.010649356966704321) + + + --> TIME: 2024-06-25 09:41:07 -- STEP: 178/591 -- GLOBAL_STEP: 25000 + | > loss_disc: 2.6749508380889893 (2.7013475948505183) + | > loss_disc_real_0: 0.23098409175872803 (0.20482111646887963) + | > loss_disc_real_1: 0.2295033037662506 (0.22541426164045764) + | > loss_disc_real_2: 0.23171855509281158 (0.2355442513408286) + | > loss_disc_real_3: 0.2393183410167694 (0.23515734855043754) + | > loss_disc_real_4: 0.23645125329494476 (0.23313959545633767) + | > loss_disc_real_5: 0.2134045958518982 (0.239469280068794) + | > loss_0: 2.6749508380889893 (2.7013475948505183) + | > grad_norm_0: tensor(5.1184, device='cuda:0') (tensor(10.9130, device='cuda:0')) + | > loss_gen: 2.0407662391662598 (1.946668351634165) + | > loss_kl: 2.459402561187744 (2.556783149751386) + | > loss_feat: 3.9983110427856445 (3.990299508812722) + | > loss_mel: 19.833118438720703 (20.6517623194148) + | > loss_duration: 1.3215657472610474 (1.3427539574965999) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.65316390991211 (30.488267191340416) + | > grad_norm_1: tensor(181.8277, device='cuda:0') (tensor(219.0006, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.8827 (0.8289067771997345) + | > loader_time: 0.0135 (0.010837704947825221) + + + > CHECKPOINT : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/checkpoint_25000.pth + + --> TIME: 2024-06-25 09:41:32 -- STEP: 203/591 -- GLOBAL_STEP: 25025 + | > loss_disc: 2.7004754543304443 (2.7008705785121823) + | > loss_disc_real_0: 0.17284035682678223 (0.2049126124073719) + | > loss_disc_real_1: 0.18826796114444733 (0.2254426753139261) + | > loss_disc_real_2: 0.21045996248722076 (0.23539972709023893) + | > loss_disc_real_3: 0.23398418724536896 (0.23518051390577419) + | > loss_disc_real_4: 0.24231509864330292 (0.2332841540351877) + | > loss_disc_real_5: 0.24497219920158386 (0.23932399178667021) + | > loss_0: 2.7004754543304443 (2.7008705785121823) + | > grad_norm_0: tensor(4.4343, device='cuda:0') (tensor(10.7870, device='cuda:0')) + | > loss_gen: 1.9400734901428223 (1.9446307855286622) + | > loss_kl: 2.4597432613372803 (2.552201554105788) + | > loss_feat: 3.8198113441467285 (3.9819039593776457) + | > loss_mel: 20.469282150268555 (20.649050566950454) + | > loss_duration: 1.339951992034912 (1.3422311702972562) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.02886199951172 (30.47001795463374) + | > grad_norm_1: tensor(58.3766, device='cuda:0') (tensor(212.9564, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.8633 (0.83378031336028) + | > loader_time: 0.0129 (0.011076115622309035) + + + --> TIME: 2024-06-25 09:41:54 -- STEP: 228/591 -- GLOBAL_STEP: 25050 + | > loss_disc: 2.664949417114258 (2.7005594791027536) + | > loss_disc_real_0: 0.20543259382247925 (0.20455141930857249) + | > loss_disc_real_1: 0.22184522449970245 (0.22545693403011874) + | > loss_disc_real_2: 0.2348475605249405 (0.23538605032259957) + | > loss_disc_real_3: 0.24768219888210297 (0.23523852765037304) + | > loss_disc_real_4: 0.2611587345600128 (0.23342447937057728) + | > loss_disc_real_5: 0.2605498433113098 (0.239217679228699) + | > loss_0: 2.664949417114258 (2.7005594791027536) + | > grad_norm_0: tensor(8.0694, device='cuda:0') (tensor(10.4734, device='cuda:0')) + | > loss_gen: 1.9419753551483154 (1.9426817815554769) + | > loss_kl: 2.5230202674865723 (2.550203742688163) + | > loss_feat: 3.929471969604492 (3.9722145001093545) + | > loss_mel: 20.4196834564209 (20.64681653809129) + | > loss_duration: 1.3256416320800781 (1.3418177031634146) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.13979148864746 (30.453734155286824) + | > grad_norm_1: tensor(47.2430, device='cuda:0') (tensor(200.7105, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.8858 (0.8384519920014498) + | > loader_time: 0.0154 (0.011431783960576637) + + + --> TIME: 2024-06-25 09:42:17 -- STEP: 253/591 -- GLOBAL_STEP: 25075 + | > loss_disc: 2.677009105682373 (2.7004490096578486) + | > loss_disc_real_0: 0.16437876224517822 (0.20428998815447916) + | > loss_disc_real_1: 0.20633238554000854 (0.22565851410622653) + | > loss_disc_real_2: 0.26808467507362366 (0.23536473229939758) + | > loss_disc_real_3: 0.21797575056552887 (0.23507359034930292) + | > loss_disc_real_4: 0.2118452787399292 (0.23344866083309113) + | > loss_disc_real_5: 0.24914374947547913 (0.2393888181495101) + | > loss_0: 2.677009105682373 (2.7004490096578486) + | > grad_norm_0: tensor(9.4479, device='cuda:0') (tensor(10.4715, device='cuda:0')) + | > loss_gen: 1.9965204000473022 (1.94286214928382) + | > loss_kl: 2.4628820419311523 (2.5497338300637113) + | > loss_feat: 3.8483173847198486 (3.9703029163270127) + | > loss_mel: 20.958948135375977 (20.649263381958008) + | > loss_duration: 1.337550401687622 (1.3427673625380625) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.604219436645508 (30.45492956289661) + | > grad_norm_1: tensor(255.0963, device='cuda:0') (tensor(197.9049, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.8901 (0.8441383782111609) + | > loader_time: 0.0129 (0.011741443113847207) + + + --> TIME: 2024-06-25 09:42:40 -- STEP: 278/591 -- GLOBAL_STEP: 25100 + | > loss_disc: 2.6613237857818604 (2.701746965483796) + | > loss_disc_real_0: 0.23628570139408112 (0.20438934693769584) + | > loss_disc_real_1: 0.23598220944404602 (0.2257635148523523) + | > loss_disc_real_2: 0.20041219890117645 (0.2359163264790885) + | > loss_disc_real_3: 0.20831400156021118 (0.23502876068190706) + | > loss_disc_real_4: 0.24783392250537872 (0.23365924426977583) + | > loss_disc_real_5: 0.24797466397285461 (0.23943481685446322) + | > loss_0: 2.6613237857818604 (2.701746965483796) + | > grad_norm_0: tensor(9.1991, device='cuda:0') (tensor(10.6697, device='cuda:0')) + | > loss_gen: 1.883903980255127 (1.9424168175930598) + | > loss_kl: 2.550107002258301 (2.54731210444471) + | > loss_feat: 4.256953716278076 (3.9665396196379077) + | > loss_mel: 21.158306121826172 (20.652052776418998) + | > loss_duration: 1.3413102626800537 (1.3431592635113558) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.190580368041992 (30.4514805018473) + | > grad_norm_1: tensor(42.9477, device='cuda:0') (tensor(188.4434, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.9067 (0.8494031489324226) + | > loader_time: 0.0153 (0.012039635678847057) + + + --> TIME: 2024-06-25 09:43:04 -- STEP: 303/591 -- GLOBAL_STEP: 25125 + | > loss_disc: 2.6902337074279785 (2.6997897231539483) + | > loss_disc_real_0: 0.15179069340229034 (0.2034398437579079) + | > loss_disc_real_1: 0.2102462649345398 (0.22567427664896836) + | > loss_disc_real_2: 0.2219335287809372 (0.2361289534730093) + | > loss_disc_real_3: 0.21716775000095367 (0.2349911967028092) + | > loss_disc_real_4: 0.23962406814098358 (0.23341375068075979) + | > loss_disc_real_5: 0.2214953601360321 (0.23921260154089913) + | > loss_0: 2.6902337074279785 (2.6997897231539483) + | > grad_norm_0: tensor(13.3212, device='cuda:0') (tensor(10.4578, device='cuda:0')) + | > loss_gen: 1.9985510110855103 (1.94413162418718) + | > loss_kl: 2.5857396125793457 (2.5453161925765944) + | > loss_feat: 3.7250945568084717 (3.966458625919355) + | > loss_mel: 21.3293514251709 (20.659366060011443) + | > loss_duration: 1.3815290927886963 (1.343647457192046) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.020265579223633 (30.458919902839284) + | > grad_norm_1: tensor(68.7948, device='cuda:0') (tensor(189.7053, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.906 (0.8558868732389444) + | > loader_time: 0.0166 (0.012290173237866688) + + + --> TIME: 2024-06-25 09:43:28 -- STEP: 328/591 -- GLOBAL_STEP: 25150 + | > loss_disc: 2.722726345062256 (2.6989532069462103) + | > loss_disc_real_0: 0.26551327109336853 (0.20324931121090564) + | > loss_disc_real_1: 0.19475236535072327 (0.22575752964107001) + | > loss_disc_real_2: 0.2606218755245209 (0.23638249170489428) + | > loss_disc_real_3: 0.28093764185905457 (0.23495501320718265) + | > loss_disc_real_4: 0.2111978530883789 (0.23320116338933386) + | > loss_disc_real_5: 0.23392024636268616 (0.23890953679092047) + | > loss_0: 2.722726345062256 (2.6989532069462103) + | > grad_norm_0: tensor(13.0532, device='cuda:0') (tensor(10.5502, device='cuda:0')) + | > loss_gen: 1.918824315071106 (1.945325931034437) + | > loss_kl: 2.5870044231414795 (2.542787261852405) + | > loss_feat: 4.092457294464111 (3.968747259640112) + | > loss_mel: 20.509042739868164 (20.66918362640751) + | > loss_duration: 1.3465979099273682 (1.3444588809478568) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.45392608642578 (30.470502940619863) + | > grad_norm_1: tensor(72.5739, device='cuda:0') (tensor(191.8528, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.9701 (0.8613125914480628) + | > loader_time: 0.0145 (0.012541699700239222) + + + --> TIME: 2024-06-25 09:43:52 -- STEP: 353/591 -- GLOBAL_STEP: 25175 + | > loss_disc: 2.664912223815918 (2.69811422764073) + | > loss_disc_real_0: 0.20442917943000793 (0.20341143419401503) + | > loss_disc_real_1: 0.19547568261623383 (0.22558280255740493) + | > loss_disc_real_2: 0.24420110881328583 (0.23620658846988893) + | > loss_disc_real_3: 0.227852001786232 (0.23480989693244522) + | > loss_disc_real_4: 0.24495047330856323 (0.23307627659006092) + | > loss_disc_real_5: 0.2393978238105774 (0.23893536686390723) + | > loss_0: 2.664912223815918 (2.69811422764073) + | > grad_norm_0: tensor(4.1608, device='cuda:0') (tensor(10.6453, device='cuda:0')) + | > loss_gen: 1.90232515335083 (1.9454063231141303) + | > loss_kl: 2.494255304336548 (2.5409282773479873) + | > loss_feat: 4.068549633026123 (3.9666891266874145) + | > loss_mel: 20.321001052856445 (20.65800255081132) + | > loss_duration: 1.3563551902770996 (1.345640715053331) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.142486572265625 (30.456667005846924) + | > grad_norm_1: tensor(124.4791, device='cuda:0') (tensor(189.9828, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.9931 (0.8674766254154886) + | > loader_time: 0.0153 (0.012794554064699336) + + + --> TIME: 2024-06-25 09:44:17 -- STEP: 378/591 -- GLOBAL_STEP: 25200 + | > loss_disc: 2.605525255203247 (2.6977668868170834) + | > loss_disc_real_0: 0.16743727028369904 (0.20306923948031255) + | > loss_disc_real_1: 0.23545482754707336 (0.22566810246340183) + | > loss_disc_real_2: 0.23941773176193237 (0.23630620578609446) + | > loss_disc_real_3: 0.22859269380569458 (0.234621341699961) + | > loss_disc_real_4: 0.2349461168050766 (0.23306876208101) + | > loss_disc_real_5: 0.24452000856399536 (0.2389567192427065) + | > loss_0: 2.605525255203247 (2.6977668868170834) + | > grad_norm_0: tensor(3.2554, device='cuda:0') (tensor(10.6944, device='cuda:0')) + | > loss_gen: 2.0913381576538086 (1.9460403417784071) + | > loss_kl: 2.579054594039917 (2.538998615804805) + | > loss_feat: 4.086943626403809 (3.9634018322778126) + | > loss_mel: 20.693227767944336 (20.641746783382665) + | > loss_duration: 1.3735244274139404 (1.3472895310038604) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.82408905029297 (30.437477126954093) + | > grad_norm_1: tensor(98.5548, device='cuda:0') (tensor(187.3378, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.9551 (0.873357685154708) + | > loader_time: 0.0169 (0.013064485378366294) + + + --> TIME: 2024-06-25 09:44:41 -- STEP: 403/591 -- GLOBAL_STEP: 25225 + | > loss_disc: 2.6882429122924805 (2.6983601064894973) + | > loss_disc_real_0: 0.25829488039016724 (0.20326549567226737) + | > loss_disc_real_1: 0.2240717113018036 (0.22552996958899438) + | > loss_disc_real_2: 0.2316284328699112 (0.23648696779169456) + | > loss_disc_real_3: 0.2199379801750183 (0.23488997788198532) + | > loss_disc_real_4: 0.21404634416103363 (0.2330232090305158) + | > loss_disc_real_5: 0.22146053612232208 (0.23883738037818122) + | > loss_0: 2.6882429122924805 (2.6983601064894973) + | > grad_norm_0: tensor(10.9881, device='cuda:0') (tensor(10.6674, device='cuda:0')) + | > loss_gen: 1.9049705266952515 (1.9456283179465357) + | > loss_kl: 2.4331793785095215 (2.537627669185328) + | > loss_feat: 4.074280261993408 (3.9579923200252334) + | > loss_mel: 21.11881446838379 (20.637619842074923) + | > loss_duration: 1.398855209350586 (1.348239531883826) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.930099487304688 (30.427107685552933) + | > grad_norm_1: tensor(274.9006, device='cuda:0') (tensor(187.4567, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.9527 (0.8794874517852556) + | > loader_time: 0.0156 (0.013320586225826742) + + + --> TIME: 2024-06-25 09:45:07 -- STEP: 428/591 -- GLOBAL_STEP: 25250 + | > loss_disc: 2.8061187267303467 (2.6979349617646107) + | > loss_disc_real_0: 0.12333925068378448 (0.20286023211186735) + | > loss_disc_real_1: 0.19577249884605408 (0.22555296948162196) + | > loss_disc_real_2: 0.24848142266273499 (0.23623000764261895) + | > loss_disc_real_3: 0.1633632779121399 (0.2347127787211788) + | > loss_disc_real_4: 0.22463761270046234 (0.23310521443452792) + | > loss_disc_real_5: 0.24135056138038635 (0.23896557312006148) + | > loss_0: 2.8061187267303467 (2.6979349617646107) + | > grad_norm_0: tensor(19.7919, device='cuda:0') (tensor(10.6674, device='cuda:0')) + | > loss_gen: 2.0731284618377686 (1.944978674159986) + | > loss_kl: 2.4236459732055664 (2.5337030408538412) + | > loss_feat: 3.9010062217712402 (3.950542990292344) + | > loss_mel: 20.880006790161133 (20.61788564753308) + | > loss_duration: 1.3706541061401367 (1.349057415099901) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.648441314697266 (30.396167746214108) + | > grad_norm_1: tensor(136.8842, device='cuda:0') (tensor(187.2836, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 0.9972 (0.8858406488026414) + | > loader_time: 0.0181 (0.013603974168545727) + + + --> TIME: 2024-06-25 09:45:33 -- STEP: 453/591 -- GLOBAL_STEP: 25275 + | > loss_disc: 2.7717459201812744 (2.6991141837164254) + | > loss_disc_real_0: 0.14265060424804688 (0.20329669386016874) + | > loss_disc_real_1: 0.245150625705719 (0.22585579167401817) + | > loss_disc_real_2: 0.24066095054149628 (0.2364275845273441) + | > loss_disc_real_3: 0.23253348469734192 (0.23498216475345704) + | > loss_disc_real_4: 0.22706376016139984 (0.23285883510481178) + | > loss_disc_real_5: 0.25233250856399536 (0.23897214654923538) + | > loss_0: 2.7717459201812744 (2.6991141837164254) + | > grad_norm_0: tensor(19.8203, device='cuda:0') (tensor(10.9044, device='cuda:0')) + | > loss_gen: 1.7375898361206055 (1.9446764492304383) + | > loss_kl: 2.552053451538086 (2.5306271448830127) + | > loss_feat: 3.268592119216919 (3.9397509024369533) + | > loss_mel: 19.711156845092773 (20.60502555459802) + | > loss_duration: 1.3723018169403076 (1.3500315429885388) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.641695022583008 (30.370111579137134) + | > grad_norm_1: tensor(130.4721, device='cuda:0') (tensor(194.8885, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 1.0175 (0.8929523395386753) + | > loader_time: 0.0238 (0.013884965420821904) + + + --> TIME: 2024-06-25 09:45:59 -- STEP: 478/591 -- GLOBAL_STEP: 25300 + | > loss_disc: 2.662698745727539 (2.6989463472964865) + | > loss_disc_real_0: 0.24761293828487396 (0.20336332170596683) + | > loss_disc_real_1: 0.21513311564922333 (0.22570966951019095) + | > loss_disc_real_2: 0.2512781620025635 (0.23642525690993504) + | > loss_disc_real_3: 0.23396478593349457 (0.2348953366404298) + | > loss_disc_real_4: 0.22394104301929474 (0.23287869556553692) + | > loss_disc_real_5: 0.2200702577829361 (0.2389692940347863) + | > loss_0: 2.662698745727539 (2.6989463472964865) + | > grad_norm_0: tensor(9.5117, device='cuda:0') (tensor(10.8815, device='cuda:0')) + | > loss_gen: 1.8474198579788208 (1.94374302141836) + | > loss_kl: 2.595438241958618 (2.5290065220709144) + | > loss_feat: 3.7377982139587402 (3.929962554237334) + | > loss_mel: 20.386032104492188 (20.59419580084508) + | > loss_duration: 1.3604656457901 (1.3511879673562783) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.927154541015625 (30.348095837996095) + | > grad_norm_1: tensor(334.7510, device='cuda:0') (tensor(197.9819, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 1.0319 (0.8998794954690973) + | > loader_time: 0.0184 (0.01414187233817128) + + + --> TIME: 2024-06-25 09:46:26 -- STEP: 503/591 -- GLOBAL_STEP: 25325 + | > loss_disc: 2.6827144622802734 (2.6989988835147067) + | > loss_disc_real_0: 0.2388089895248413 (0.2034653584745011) + | > loss_disc_real_1: 0.19174543023109436 (0.2256943081648165) + | > loss_disc_real_2: 0.22225669026374817 (0.2362584990428883) + | > loss_disc_real_3: 0.26457521319389343 (0.2348305131521661) + | > loss_disc_real_4: 0.20813003182411194 (0.23285207321226714) + | > loss_disc_real_5: 0.23276293277740479 (0.23897383926402976) + | > loss_0: 2.6827144622802734 (2.6989988835147067) + | > grad_norm_0: tensor(10.5981, device='cuda:0') (tensor(10.9215, device='cuda:0')) + | > loss_gen: 1.8559033870697021 (1.9429629197414542) + | > loss_kl: 2.429318428039551 (2.5276992325754355) + | > loss_feat: 3.707590103149414 (3.9237870786109452) + | > loss_mel: 20.29483413696289 (20.576005829494456) + | > loss_duration: 1.406810998916626 (1.3526904706215521) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.6944580078125 (30.32314553080686) + | > grad_norm_1: tensor(348.4096, device='cuda:0') (tensor(197.9426, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 1.0668 (0.9083958800222954) + | > loader_time: 0.0175 (0.014432725091103764) + + + --> TIME: 2024-06-25 09:46:55 -- STEP: 528/591 -- GLOBAL_STEP: 25350 + | > loss_disc: 2.720287799835205 (2.6987231929193833) + | > loss_disc_real_0: 0.23276597261428833 (0.20330658807616794) + | > loss_disc_real_1: 0.24165354669094086 (0.22566129721588257) + | > loss_disc_real_2: 0.20141243934631348 (0.2362353806807236) + | > loss_disc_real_3: 0.22768059372901917 (0.23477030314053549) + | > loss_disc_real_4: 0.2618280351161957 (0.23270688773217527) + | > loss_disc_real_5: 0.2532787322998047 (0.23892784987886748) + | > loss_0: 2.720287799835205 (2.6987231929193833) + | > grad_norm_0: tensor(8.3877, device='cuda:0') (tensor(10.8103, device='cuda:0')) + | > loss_gen: 2.043760061264038 (1.9435464432745269) + | > loss_kl: 2.5214481353759766 (2.5250833364147147) + | > loss_feat: 3.689453363418579 (3.9201110642064703) + | > loss_mel: 20.561208724975586 (20.571757760914878) + | > loss_duration: 1.424303650856018 (1.354539163184888) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.24017333984375 (30.31503777070479) + | > grad_norm_1: tensor(422.2361, device='cuda:0') (tensor(204.5496, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 1.1188 (0.9183957992178021) + | > loader_time: 0.0198 (0.014709210757053255) + + + --> TIME: 2024-06-25 09:47:25 -- STEP: 553/591 -- GLOBAL_STEP: 25375 + | > loss_disc: 2.713736057281494 (2.6994563882217264) + | > loss_disc_real_0: 0.1896938979625702 (0.2035075134680241) + | > loss_disc_real_1: 0.21284736692905426 (0.2257364750932397) + | > loss_disc_real_2: 0.23011967539787292 (0.23613447778155963) + | > loss_disc_real_3: 0.22655978798866272 (0.2348286228197177) + | > loss_disc_real_4: 0.22254642844200134 (0.23275480027332437) + | > loss_disc_real_5: 0.22192373871803284 (0.23897283983704837) + | > loss_0: 2.713736057281494 (2.6994563882217264) + | > grad_norm_0: tensor(5.7814, device='cuda:0') (tensor(10.8468, device='cuda:0')) + | > loss_gen: 1.9412519931793213 (1.9425635316272756) + | > loss_kl: 2.4253647327423096 (2.5222793996441872) + | > loss_feat: 3.7698323726654053 (3.9108793813108753) + | > loss_mel: 19.942365646362305 (20.557320389566538) + | > loss_duration: 1.402745008468628 (1.3562898791073452) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.481557846069336 (30.289332557112566) + | > grad_norm_1: tensor(64.1533, device='cuda:0') (tensor(204.9101, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 1.2566 (0.9306151634099039) + | > loader_time: 0.0272 (0.015008105507379842) + + + --> TIME: 2024-06-25 09:47:57 -- STEP: 578/591 -- GLOBAL_STEP: 25400 + | > loss_disc: 2.7031829357147217 (2.699845291338989) + | > loss_disc_real_0: 0.15380412340164185 (0.20355633175754625) + | > loss_disc_real_1: 0.1910305768251419 (0.22572979552729325) + | > loss_disc_real_2: 0.2525350749492645 (0.23608142694387468) + | > loss_disc_real_3: 0.2130867838859558 (0.23485993521745643) + | > loss_disc_real_4: 0.22742076218128204 (0.23281468206107817) + | > loss_disc_real_5: 0.24923105537891388 (0.23903901062931565) + | > loss_0: 2.7031829357147217 (2.699845291338989) + | > grad_norm_0: tensor(9.9763, device='cuda:0') (tensor(10.8269, device='cuda:0')) + | > loss_gen: 2.0603020191192627 (1.9416622424620658) + | > loss_kl: 2.466038227081299 (2.520218511766217) + | > loss_feat: 3.9224307537078857 (3.9031444189870235) + | > loss_mel: 20.00031852722168 (20.544375884904163) + | > loss_duration: 1.4001963138580322 (1.3580986220118902) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.849285125732422 (30.267499656413246) + | > grad_norm_1: tensor(185.8578, device='cuda:0') (tensor(199.8170, device='cuda:0')) + | > current_lr_0: 0.00019895268614608487 + | > current_lr_1: 0.00019895268614608487 + | > step_time: 1.1855 (0.9435750992652867) + | > loader_time: 0.0197 (0.015327845890216758) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036606433543753115 (+3.542291357162136e-05) + | > avg_loss_disc: 2.66967518278893 (-0.11925074901986621) + | > avg_loss_disc_real_0: 0.16763408894234516 (-0.22639836149012782) + | > avg_loss_disc_real_1: 0.245880717609791 (+0.03995988628965738) + | > avg_loss_disc_real_2: 0.24306729277397723 (+0.0079727902057323) + | > avg_loss_disc_real_3: 0.24931180857597518 (+0.007167312693088651) + | > avg_loss_disc_real_4: 0.21519581371165336 (-0.02417958987520097) + | > avg_loss_disc_real_5: 0.21466218061903689 (-0.02868349438017989) + | > avg_loss_0: 2.66967518278893 (-0.11925074901986621) + | > avg_loss_gen: 1.8748443304224218 (-0.22496487231964846) + | > avg_loss_kl: 2.5428878398651773 (-0.0035734937546108014) + | > avg_loss_feat: 3.8435073710502463 (+0.12674499065318034) + | > avg_loss_mel: 20.523396025312707 (+0.18515396118164062) + | > avg_loss_duration: 1.3794529133654656 (-0.0003579525237387138) + | > avg_loss_1: 30.164088675316343 (+0.08300261801861453) + + + > EPOCH: 44/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 09:48:21)  + + --> TIME: 2024-06-25 09:48:32 -- STEP: 12/591 -- GLOBAL_STEP: 25425 + | > loss_disc: 2.7242698669433594 (2.6732760866483054) + | > loss_disc_real_0: 0.157007098197937 (0.20064657181501389) + | > loss_disc_real_1: 0.19728578627109528 (0.22391525655984879) + | > loss_disc_real_2: 0.22226540744304657 (0.2270156890153885) + | > loss_disc_real_3: 0.21268773078918457 (0.23384992281595865) + | > loss_disc_real_4: 0.22357705235481262 (0.23110835005839667) + | > loss_disc_real_5: 0.23987692594528198 (0.2438328117132187) + | > loss_0: 2.7242698669433594 (2.6732760866483054) + | > grad_norm_0: tensor(10.8318, device='cuda:0') (tensor(9.2041, device='cuda:0')) + | > loss_gen: 2.1860263347625732 (1.9988625744978588) + | > loss_kl: 2.605398178100586 (2.6806530356407166) + | > loss_feat: 4.315256118774414 (4.2643989721933995) + | > loss_mel: 21.004854202270508 (20.821685473124187) + | > loss_duration: 1.309099793434143 (1.3261440594991047) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.42063331604004 (31.09174378712972) + | > grad_norm_1: tensor(53.4707, device='cuda:0') (tensor(100.7546, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.7687 (0.7718750238418579) + | > loader_time: 0.0092 (0.008710245291392008) + + + --> TIME: 2024-06-25 09:48:52 -- STEP: 37/591 -- GLOBAL_STEP: 25450 + | > loss_disc: 2.6595091819763184 (2.6958607403007715) + | > loss_disc_real_0: 0.20450080931186676 (0.20695621177956863) + | > loss_disc_real_1: 0.2012583464384079 (0.22578954575835047) + | > loss_disc_real_2: 0.23958392441272736 (0.23212589806801565) + | > loss_disc_real_3: 0.22865335643291473 (0.23483149908684395) + | > loss_disc_real_4: 0.217018723487854 (0.23372614101783648) + | > loss_disc_real_5: 0.24376949667930603 (0.23984853117852598) + | > loss_0: 2.6595091819763184 (2.6958607403007715) + | > grad_norm_0: tensor(5.6215, device='cuda:0') (tensor(10.2020, device='cuda:0')) + | > loss_gen: 1.8707330226898193 (1.9577195934347205) + | > loss_kl: 2.573941469192505 (2.623404586637342) + | > loss_feat: 3.9394030570983887 (4.079126113169902) + | > loss_mel: 20.232349395751953 (20.63247402294262) + | > loss_duration: 1.3247469663619995 (1.3254686207384674) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.941173553466797 (30.618192878929346) + | > grad_norm_1: tensor(68.2313, device='cuda:0') (tensor(104.5907, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.7957 (0.7852208549911911) + | > loader_time: 0.0103 (0.009396230852281723) + + + --> TIME: 2024-06-25 09:49:13 -- STEP: 62/591 -- GLOBAL_STEP: 25475 + | > loss_disc: 2.909846067428589 (2.6961455691245293) + | > loss_disc_real_0: 0.4746655821800232 (0.210029337915682) + | > loss_disc_real_1: 0.24579016864299774 (0.22646076833048173) + | > loss_disc_real_2: 0.22654616832733154 (0.2340120814980999) + | > loss_disc_real_3: 0.3317531943321228 (0.2352404099318289) + | > loss_disc_real_4: 0.2215138077735901 (0.23045282306209688) + | > loss_disc_real_5: 0.22568359971046448 (0.23877681263031497) + | > loss_0: 2.909846067428589 (2.6961455691245293) + | > grad_norm_0: tensor(50.0100, device='cuda:0') (tensor(12.0398, device='cuda:0')) + | > loss_gen: 1.7549220323562622 (1.9733090323786582) + | > loss_kl: 2.5987818241119385 (2.6013325222076906) + | > loss_feat: 3.960724353790283 (4.093736402450069) + | > loss_mel: 20.920316696166992 (20.65073714717742) + | > loss_duration: 1.360063076019287 (1.3333017845307622) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.594810485839844 (30.65241696757655) + | > grad_norm_1: tensor(200.6965, device='cuda:0') (tensor(186.1855, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.8212 (0.7954648502411381) + | > loader_time: 0.011 (0.009748085852592219) + + + --> TIME: 2024-06-25 09:49:34 -- STEP: 87/591 -- GLOBAL_STEP: 25500 + | > loss_disc: 2.693293571472168 (2.691637444770199) + | > loss_disc_real_0: 0.19167229533195496 (0.2060155444617929) + | > loss_disc_real_1: 0.25173628330230713 (0.22634300503922605) + | > loss_disc_real_2: 0.25615400075912476 (0.23438823565669442) + | > loss_disc_real_3: 0.202616885304451 (0.23479368066650697) + | > loss_disc_real_4: 0.20786930620670319 (0.22966365550441303) + | > loss_disc_real_5: 0.23077347874641418 (0.23785504527475643) + | > loss_0: 2.693293571472168 (2.691637444770199) + | > grad_norm_0: tensor(5.4093, device='cuda:0') (tensor(11.8392, device='cuda:0')) + | > loss_gen: 1.7831578254699707 (1.9714822399205174) + | > loss_kl: 2.5610761642456055 (2.5851783533205936) + | > loss_feat: 3.972851514816284 (4.084069038259571) + | > loss_mel: 20.620901107788086 (20.67769611840961) + | > loss_duration: 1.3641011714935303 (1.3402456565835008) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.302087783813477 (30.65867147774532) + | > grad_norm_1: tensor(137.1704, device='cuda:0') (tensor(211.2640, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.8192 (0.8032004011088404) + | > loader_time: 0.0106 (0.009971393935981834) + + + --> TIME: 2024-06-25 09:49:55 -- STEP: 112/591 -- GLOBAL_STEP: 25525 + | > loss_disc: 2.6361727714538574 (2.6928339856011516) + | > loss_disc_real_0: 0.18527068197727203 (0.20523768603535636) + | > loss_disc_real_1: 0.2475575953722 (0.226169278845191) + | > loss_disc_real_2: 0.22087650001049042 (0.23436731871749675) + | > loss_disc_real_3: 0.24168099462985992 (0.2348731601876872) + | > loss_disc_real_4: 0.21065042912960052 (0.23085159874920333) + | > loss_disc_real_5: 0.20601466298103333 (0.2380118883614029) + | > loss_0: 2.6361727714538574 (2.6928339856011516) + | > grad_norm_0: tensor(4.6905, device='cuda:0') (tensor(11.1324, device='cuda:0')) + | > loss_gen: 2.0293569564819336 (1.9682603180408478) + | > loss_kl: 2.538045883178711 (2.567530276519913) + | > loss_feat: 4.215351581573486 (4.0757917889526905) + | > loss_mel: 21.140588760375977 (20.708835601806648) + | > loss_duration: 1.371262788772583 (1.344124717371804) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.294607162475586 (30.6645427942276) + | > grad_norm_1: tensor(199.6786, device='cuda:0') (tensor(198.0577, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.8288 (0.8095981457403728) + | > loader_time: 0.0105 (0.010188698768615714) + + + --> TIME: 2024-06-25 09:50:17 -- STEP: 137/591 -- GLOBAL_STEP: 25550 + | > loss_disc: 2.6617908477783203 (2.6926142090428478) + | > loss_disc_real_0: 0.23221418261528015 (0.20536676783413782) + | > loss_disc_real_1: 0.2537783086299896 (0.22595876303032367) + | > loss_disc_real_2: 0.23172436654567719 (0.23433604966985047) + | > loss_disc_real_3: 0.22187121212482452 (0.23466363755890923) + | > loss_disc_real_4: 0.2116043120622635 (0.2315400423359697) + | > loss_disc_real_5: 0.22994071245193481 (0.23863113307169753) + | > loss_0: 2.6617908477783203 (2.6926142090428478) + | > grad_norm_0: tensor(8.5573, device='cuda:0') (tensor(10.5716, device='cuda:0')) + | > loss_gen: 1.8585833311080933 (1.9636975944477275) + | > loss_kl: 2.5388362407684326 (2.559724527553921) + | > loss_feat: 4.08699893951416 (4.066672669710034) + | > loss_mel: 20.53168296813965 (20.69609441548369) + | > loss_duration: 1.3292299509048462 (1.3443535688149666) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.3453311920166 (30.63054289434948) + | > grad_norm_1: tensor(158.9616, device='cuda:0') (tensor(184.2555, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.8373 (0.8163665437350308) + | > loader_time: 0.0108 (0.010389963205713419) + + + --> TIME: 2024-06-25 09:50:38 -- STEP: 162/591 -- GLOBAL_STEP: 25575 + | > loss_disc: 2.7496285438537598 (2.693851753517432) + | > loss_disc_real_0: 0.2577659487724304 (0.20638890063137183) + | > loss_disc_real_1: 0.2570739984512329 (0.22559379878235453) + | > loss_disc_real_2: 0.2546484172344208 (0.23419998908116493) + | > loss_disc_real_3: 0.25983893871307373 (0.2348359371592969) + | > loss_disc_real_4: 0.2615800201892853 (0.23189096907038748) + | > loss_disc_real_5: 0.26067620515823364 (0.2385176831373462) + | > loss_0: 2.7496285438537598 (2.693851753517432) + | > grad_norm_0: tensor(8.3378, device='cuda:0') (tensor(10.6991, device='cuda:0')) + | > loss_gen: 1.8402142524719238 (1.9632457980403193) + | > loss_kl: 2.499978542327881 (2.5526908562507167) + | > loss_feat: 4.059709548950195 (4.063350281597655) + | > loss_mel: 19.73419189453125 (20.675642237251196) + | > loss_duration: 1.3343322277069092 (1.3430997648356875) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.468425750732422 (30.5980290306939) + | > grad_norm_1: tensor(36.6958, device='cuda:0') (tensor(183.0624, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.8714 (0.8227011333277197) + | > loader_time: 0.0114 (0.010590616567635233) + + + --> TIME: 2024-06-25 09:51:01 -- STEP: 187/591 -- GLOBAL_STEP: 25600 + | > loss_disc: 2.6209566593170166 (2.6919115584164364) + | > loss_disc_real_0: 0.16851739585399628 (0.20560695728994308) + | > loss_disc_real_1: 0.17573466897010803 (0.22532302475868063) + | > loss_disc_real_2: 0.22304798662662506 (0.23379639261227878) + | > loss_disc_real_3: 0.2322636991739273 (0.23461641610306214) + | > loss_disc_real_4: 0.2203274369239807 (0.23172596495419262) + | > loss_disc_real_5: 0.2522193193435669 (0.23860318528458396) + | > loss_0: 2.6209566593170166 (2.6919115584164364) + | > grad_norm_0: tensor(4.7713, device='cuda:0') (tensor(10.6127, device='cuda:0')) + | > loss_gen: 2.0094027519226074 (1.9630461835606212) + | > loss_kl: 2.6525087356567383 (2.547938382561832) + | > loss_feat: 4.32615327835083 (4.067025106858442) + | > loss_mel: 20.84735870361328 (20.67664402436446) + | > loss_duration: 1.3152079582214355 (1.3420513466717723) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.150632858276367 (30.596705095015746) + | > grad_norm_1: tensor(148.2863, device='cuda:0') (tensor(183.5423, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.8608 (0.8284869206780419) + | > loader_time: 0.0118 (0.010787358258497257) + + + --> TIME: 2024-06-25 09:51:23 -- STEP: 212/591 -- GLOBAL_STEP: 25625 + | > loss_disc: 2.640108108520508 (2.6928538117768617) + | > loss_disc_real_0: 0.2347993552684784 (0.20554567244395894) + | > loss_disc_real_1: 0.22021342813968658 (0.22527939341259454) + | > loss_disc_real_2: 0.23197007179260254 (0.2345808964433535) + | > loss_disc_real_3: 0.20852643251419067 (0.23438293051044895) + | > loss_disc_real_4: 0.23438987135887146 (0.23201819129709927) + | > loss_disc_real_5: 0.24765853583812714 (0.2387086735019144) + | > loss_0: 2.640108108520508 (2.6928538117768617) + | > grad_norm_0: tensor(9.1890, device='cuda:0') (tensor(10.5311, device='cuda:0')) + | > loss_gen: 1.9740506410598755 (1.9602349760397426) + | > loss_kl: 2.491237163543701 (2.543646500920351) + | > loss_feat: 4.072767734527588 (4.05627182969507) + | > loss_mel: 21.313051223754883 (20.677804128179027) + | > loss_duration: 1.3229109048843384 (1.3411512402993322) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.17401695251465 (30.5791087240543) + | > grad_norm_1: tensor(154.3603, device='cuda:0') (tensor(185.9864, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.8814 (0.8353904676887224) + | > loader_time: 0.0135 (0.011049912785584062) + + + --> TIME: 2024-06-25 09:51:46 -- STEP: 237/591 -- GLOBAL_STEP: 25650 + | > loss_disc: 2.7454631328582764 (2.692912036356543) + | > loss_disc_real_0: 0.2227843701839447 (0.20496105217229466) + | > loss_disc_real_1: 0.22412842512130737 (0.22497642907915236) + | > loss_disc_real_2: 0.23224180936813354 (0.23499274058935513) + | > loss_disc_real_3: 0.2934446930885315 (0.2347735092986988) + | > loss_disc_real_4: 0.22111961245536804 (0.23201497446132613) + | > loss_disc_real_5: 0.20077431201934814 (0.23867417412971142) + | > loss_0: 2.7454631328582764 (2.692912036356543) + | > grad_norm_0: tensor(4.4363, device='cuda:0') (tensor(10.2184, device='cuda:0')) + | > loss_gen: 2.025021553039551 (1.9595414193873668) + | > loss_kl: 2.516657590866089 (2.5402000523820716) + | > loss_feat: 3.9563987255096436 (4.0529413565301695) + | > loss_mel: 20.679149627685547 (20.68053635062046) + | > loss_duration: 1.3351285457611084 (1.3410200700478208) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.51235580444336 (30.574239256009776) + | > grad_norm_1: tensor(129.7478, device='cuda:0') (tensor(179.2801, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.9174 (0.8404377734107811) + | > loader_time: 0.0151 (0.011380037677941936) + + + --> TIME: 2024-06-25 09:52:09 -- STEP: 262/591 -- GLOBAL_STEP: 25675 + | > loss_disc: 2.6954455375671387 (2.6937251127403194) + | > loss_disc_real_0: 0.11999009549617767 (0.20453104206168923) + | > loss_disc_real_1: 0.23807492852210999 (0.2252088992195275) + | > loss_disc_real_2: 0.25513485074043274 (0.23516641250093476) + | > loss_disc_real_3: 0.2406957745552063 (0.23473871425362944) + | > loss_disc_real_4: 0.22199547290802002 (0.2323434069070197) + | > loss_disc_real_5: 0.2324945479631424 (0.23877611574325852) + | > loss_0: 2.6954455375671387 (2.6937251127403194) + | > grad_norm_0: tensor(18.5741, device='cuda:0') (tensor(10.3544, device='cuda:0')) + | > loss_gen: 1.9805150032043457 (1.9586120124991613) + | > loss_kl: 2.6538922786712646 (2.536379710408568) + | > loss_feat: 4.1548614501953125 (4.049362248136797) + | > loss_mel: 20.100547790527344 (20.65725453937328) + | > loss_duration: 1.3260018825531006 (1.3408975396447504) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.215818405151367 (30.542506057797496) + | > grad_norm_1: tensor(57.2793, device='cuda:0') (tensor(174.3949, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.8996 (0.8458377523276642) + | > loader_time: 0.015 (0.011667940452808634) + + + --> TIME: 2024-06-25 09:52:32 -- STEP: 287/591 -- GLOBAL_STEP: 25700 + | > loss_disc: 2.6506643295288086 (2.694490932006038) + | > loss_disc_real_0: 0.1755099594593048 (0.20468804518001002) + | > loss_disc_real_1: 0.24775181710720062 (0.22542187976504868) + | > loss_disc_real_2: 0.2324822098016739 (0.23531330135433515) + | > loss_disc_real_3: 0.2327762097120285 (0.23453028931019612) + | > loss_disc_real_4: 0.2373306006193161 (0.23246504963482714) + | > loss_disc_real_5: 0.24068741500377655 (0.23865670630325422) + | > loss_0: 2.6506643295288086 (2.694490932006038) + | > grad_norm_0: tensor(6.0243, device='cuda:0') (tensor(10.4173, device='cuda:0')) + | > loss_gen: 1.8651881217956543 (1.95622394599981) + | > loss_kl: 2.409503698348999 (2.5321452866863297) + | > loss_feat: 4.068108558654785 (4.043244523038433) + | > loss_mel: 20.69331169128418 (20.644497701930682) + | > loss_duration: 1.354966640472412 (1.3417244617947297) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.391080856323242 (30.517835942710317) + | > grad_norm_1: tensor(75.3998, device='cuda:0') (tensor(172.6262, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.91 (0.8510961241838407) + | > loader_time: 0.0163 (0.011948063813850846) + + + --> TIME: 2024-06-25 09:52:55 -- STEP: 312/591 -- GLOBAL_STEP: 25725 + | > loss_disc: 2.7223942279815674 (2.6953247877267685) + | > loss_disc_real_0: 0.17339138686656952 (0.2043637197034863) + | > loss_disc_real_1: 0.20727820694446564 (0.22538351697417405) + | > loss_disc_real_2: 0.2327529340982437 (0.235347999068789) + | > loss_disc_real_3: 0.2095814049243927 (0.2346147516121467) + | > loss_disc_real_4: 0.22220972180366516 (0.23261919335868114) + | > loss_disc_real_5: 0.2610423266887665 (0.2390096087772877) + | > loss_0: 2.7223942279815674 (2.6953247877267685) + | > grad_norm_0: tensor(8.4444, device='cuda:0') (tensor(10.1135, device='cuda:0')) + | > loss_gen: 1.9425110816955566 (1.9540719722325985) + | > loss_kl: 2.5937676429748535 (2.532075826174174) + | > loss_feat: 3.826740264892578 (4.034046247219427) + | > loss_mel: 20.716073989868164 (20.64437743945001) + | > loss_duration: 1.3502910137176514 (1.342288219011746) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.42938232421875 (30.506859730451534) + | > grad_norm_1: tensor(167.0984, device='cuda:0') (tensor(173.2681, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.9236 (0.856560902717786) + | > loader_time: 0.0141 (0.012175757151383618) + + + --> TIME: 2024-06-25 09:53:19 -- STEP: 337/591 -- GLOBAL_STEP: 25750 + | > loss_disc: 2.6598148345947266 (2.6950996516366375) + | > loss_disc_real_0: 0.19677157700061798 (0.2046854593500893) + | > loss_disc_real_1: 0.2167743295431137 (0.22507780454279053) + | > loss_disc_real_2: 0.22753240168094635 (0.23529681979018077) + | > loss_disc_real_3: 0.20480477809906006 (0.23458801125914125) + | > loss_disc_real_4: 0.22693005204200745 (0.2323706049682125) + | > loss_disc_real_5: 0.24038831889629364 (0.2388053725664269) + | > loss_0: 2.6598148345947266 (2.6950996516366375) + | > grad_norm_0: tensor(10.7250, device='cuda:0') (tensor(10.3554, device='cuda:0')) + | > loss_gen: 1.8574796915054321 (1.9538630194055575) + | > loss_kl: 2.594109296798706 (2.5304995439172857) + | > loss_feat: 3.9070582389831543 (4.025265033591748) + | > loss_mel: 20.935277938842773 (20.633951164494658) + | > loss_duration: 1.3601179122924805 (1.3434154509790566) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.654041290283203 (30.486994231136332) + | > grad_norm_1: tensor(203.6368, device='cuda:0') (tensor(185.9352, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.92 (0.8617810637024107) + | > loader_time: 0.014 (0.012437042213688966) + + + --> TIME: 2024-06-25 09:53:43 -- STEP: 362/591 -- GLOBAL_STEP: 25775 + | > loss_disc: 2.766040802001953 (2.695070893724978) + | > loss_disc_real_0: 0.2822558283805847 (0.20486850001288381) + | > loss_disc_real_1: 0.2298281192779541 (0.22500030206547258) + | > loss_disc_real_2: 0.2500309944152832 (0.23528736772128891) + | > loss_disc_real_3: 0.2521374225616455 (0.23468336261273748) + | > loss_disc_real_4: 0.24496524035930634 (0.2320333075457515) + | > loss_disc_real_5: 0.24849528074264526 (0.23867582612110105) + | > loss_0: 2.766040802001953 (2.695070893724978) + | > grad_norm_0: tensor(12.9561, device='cuda:0') (tensor(10.4220, device='cuda:0')) + | > loss_gen: 1.9178836345672607 (1.9548464964107914) + | > loss_kl: 2.3492660522460938 (2.5275726733286747) + | > loss_feat: 3.7588768005371094 (4.020662343963073) + | > loss_mel: 20.44837760925293 (20.632755806432932) + | > loss_duration: 1.3842558860778809 (1.3449481274541562) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.858659744262695 (30.480785469982507) + | > grad_norm_1: tensor(401.4797, device='cuda:0') (tensor(195.8929, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.9734 (0.8675902423279062) + | > loader_time: 0.0175 (0.012650148644631733) + + + --> TIME: 2024-06-25 09:54:08 -- STEP: 387/591 -- GLOBAL_STEP: 25800 + | > loss_disc: 2.714355230331421 (2.6956539714675225) + | > loss_disc_real_0: 0.24819537997245789 (0.20462837251196844) + | > loss_disc_real_1: 0.23385582864284515 (0.2253976614823329) + | > loss_disc_real_2: 0.24118535220623016 (0.23526640170920418) + | > loss_disc_real_3: 0.2280697375535965 (0.23470750262416917) + | > loss_disc_real_4: 0.2333064079284668 (0.2317063199688298) + | > loss_disc_real_5: 0.25421127676963806 (0.2388976396901355) + | > loss_0: 2.714355230331421 (2.6956539714675225) + | > grad_norm_0: tensor(10.3800, device='cuda:0') (tensor(10.4531, device='cuda:0')) + | > loss_gen: 1.9242398738861084 (1.954513384390247) + | > loss_kl: 2.4306488037109375 (2.524609183156215) + | > loss_feat: 3.9424591064453125 (4.013334383952214) + | > loss_mel: 20.783424377441406 (20.629530031859755) + | > loss_duration: 1.3299181461334229 (1.3462231682868577) + | > amp_scaler: 256.0 (259.30749354005155) + | > loss_1: 30.410690307617188 (30.46821016612287) + | > grad_norm_1: tensor(477.1032, device='cuda:0') (tensor(207.2712, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.985 (0.873528969688317) + | > loader_time: 0.0175 (0.012895309955882775) + + + --> TIME: 2024-06-25 09:54:33 -- STEP: 412/591 -- GLOBAL_STEP: 25825 + | > loss_disc: 2.638187885284424 (2.6956098137549978) + | > loss_disc_real_0: 0.15640345215797424 (0.20444068433807314) + | > loss_disc_real_1: 0.24086356163024902 (0.22521723361993298) + | > loss_disc_real_2: 0.23435743153095245 (0.23518478935494005) + | > loss_disc_real_3: 0.22037702798843384 (0.234722440574065) + | > loss_disc_real_4: 0.25515836477279663 (0.23195451758440258) + | > loss_disc_real_5: 0.2772226929664612 (0.23903741499463332) + | > loss_0: 2.638187885284424 (2.6956098137549978) + | > grad_norm_0: tensor(8.3661, device='cuda:0') (tensor(10.3662, device='cuda:0')) + | > loss_gen: 2.0563509464263916 (1.9531425532785434) + | > loss_kl: 2.5216219425201416 (2.5220642709037637) + | > loss_feat: 3.771484613418579 (4.002675253789401) + | > loss_mel: 20.81441307067871 (20.61424872018761) + | > loss_duration: 1.3507769107818604 (1.3471165568504513) + | > amp_scaler: 256.0 (259.10679611650465) + | > loss_1: 30.514646530151367 (30.439247372080978) + | > grad_norm_1: tensor(153.0451, device='cuda:0') (tensor(207.9232, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 0.9892 (0.8805978720627942) + | > loader_time: 0.0171 (0.013162616387154288) + + + --> TIME: 2024-06-25 09:54:59 -- STEP: 437/591 -- GLOBAL_STEP: 25850 + | > loss_disc: 2.6658689975738525 (2.696784418834974) + | > loss_disc_real_0: 0.17977088689804077 (0.20458172914706324) + | > loss_disc_real_1: 0.1981537789106369 (0.22518851749547833) + | > loss_disc_real_2: 0.2308158576488495 (0.2351382785312644) + | > loss_disc_real_3: 0.2530728876590729 (0.23471564862629915) + | > loss_disc_real_4: 0.20663990080356598 (0.23217170659955633) + | > loss_disc_real_5: 0.22549177706241608 (0.23916445041684864) + | > loss_0: 2.6658689975738525 (2.696784418834974) + | > grad_norm_0: tensor(6.1551, device='cuda:0') (tensor(10.4677, device='cuda:0')) + | > loss_gen: 1.9441280364990234 (1.9509585975782278) + | > loss_kl: 2.5863044261932373 (2.5204493033804125) + | > loss_feat: 4.001802921295166 (3.992199761916626) + | > loss_mel: 20.8083553314209 (20.610164760179348) + | > loss_duration: 1.3619389533996582 (1.3482674297533532) + | > amp_scaler: 256.0 (258.9290617848969) + | > loss_1: 30.70252799987793 (30.422039843805866) + | > grad_norm_1: tensor(51.3910, device='cuda:0') (tensor(204.7552, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 1.008 (0.8879455619873117) + | > loader_time: 0.0184 (0.013440445304189717) + + + --> TIME: 2024-06-25 09:55:25 -- STEP: 462/591 -- GLOBAL_STEP: 25875 + | > loss_disc: 2.7773959636688232 (2.697800858712299) + | > loss_disc_real_0: 0.3448956608772278 (0.20495847423935865) + | > loss_disc_real_1: 0.24152252078056335 (0.22513088535565834) + | > loss_disc_real_2: 0.2384028434753418 (0.2351738748612342) + | > loss_disc_real_3: 0.32556745409965515 (0.23485100146753962) + | > loss_disc_real_4: 0.25256606936454773 (0.23238401079461687) + | > loss_disc_real_5: 0.2562350332736969 (0.23912379793919525) + | > loss_0: 2.7773959636688232 (2.697800858712299) + | > grad_norm_0: tensor(22.2391, device='cuda:0') (tensor(10.5256, device='cuda:0')) + | > loss_gen: 1.8573970794677734 (1.9491846213093051) + | > loss_kl: 2.366154670715332 (2.516491185019027) + | > loss_feat: 3.735337257385254 (3.9790060267303935) + | > loss_mel: 20.477930068969727 (20.598003858095662) + | > loss_duration: 1.3854761123657227 (1.3494839985649303) + | > amp_scaler: 256.0 (258.77056277056255) + | > loss_1: 29.822296142578125 (30.392169671657282) + | > grad_norm_1: tensor(51.0880, device='cuda:0') (tensor(199.7405, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 1.0373 (0.8949340014230638) + | > loader_time: 0.0197 (0.013728242415886417) + + + --> TIME: 2024-06-25 09:55:51 -- STEP: 487/591 -- GLOBAL_STEP: 25900 + | > loss_disc: 2.705054521560669 (2.6981380725053787) + | > loss_disc_real_0: 0.234834223985672 (0.20474211964947486) + | > loss_disc_real_1: 0.19675958156585693 (0.22531905439844857) + | > loss_disc_real_2: 0.2351321578025818 (0.2352831416918267) + | > loss_disc_real_3: 0.2627998888492584 (0.2347746522030057) + | > loss_disc_real_4: 0.24657931923866272 (0.23245154541498336) + | > loss_disc_real_5: 0.24329692125320435 (0.23917321310028647) + | > loss_0: 2.705054521560669 (2.6981380725053787) + | > grad_norm_0: tensor(6.4362, device='cuda:0') (tensor(10.4909, device='cuda:0')) + | > loss_gen: 1.8797687292099 (1.9477821833299171) + | > loss_kl: 2.494746685028076 (2.5139087832683904) + | > loss_feat: 3.8391878604888916 (3.9698586586319693) + | > loss_mel: 20.253337860107422 (20.589987228048923) + | > loss_duration: 1.368869423866272 (1.3507430372296902) + | > amp_scaler: 256.0 (258.6283367556465) + | > loss_1: 29.83591079711914 (30.372279870191882) + | > grad_norm_1: tensor(152.4285, device='cuda:0') (tensor(199.4887, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 1.058 (0.9026873283072908) + | > loader_time: 0.0221 (0.014003960748472738) + + + --> TIME: 2024-06-25 09:56:20 -- STEP: 512/591 -- GLOBAL_STEP: 25925 + | > loss_disc: 2.6859183311462402 (2.6985074561089273) + | > loss_disc_real_0: 0.1454409956932068 (0.20448942565417383) + | > loss_disc_real_1: 0.24367572367191315 (0.22546974549186416) + | > loss_disc_real_2: 0.2599843442440033 (0.23524344313773327) + | > loss_disc_real_3: 0.20649035274982452 (0.23458429201855324) + | > loss_disc_real_4: 0.2134457677602768 (0.23249149735784158) + | > loss_disc_real_5: 0.2134970873594284 (0.23925138381309807) + | > loss_0: 2.6859183311462402 (2.6985074561089273) + | > grad_norm_0: tensor(9.1298, device='cuda:0') (tensor(10.3900, device='cuda:0')) + | > loss_gen: 2.0602591037750244 (1.9464047392830253) + | > loss_kl: 2.369300603866577 (2.510702959261835) + | > loss_feat: 3.8034958839416504 (3.960606679785997) + | > loss_mel: 19.945398330688477 (20.58143972605469) + | > loss_duration: 1.4257209300994873 (1.3524913734290744) + | > amp_scaler: 256.0 (258.4999999999998) + | > loss_1: 29.60417366027832 (30.351645454764366) + | > grad_norm_1: tensor(191.4546, device='cuda:0') (tensor(194.6825, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 1.0698 (0.9126773360185325) + | > loader_time: 0.0187 (0.014319917652755972) + + + --> TIME: 2024-06-25 09:56:50 -- STEP: 537/591 -- GLOBAL_STEP: 25950 + | > loss_disc: 2.7697701454162598 (2.700869450578032) + | > loss_disc_real_0: 0.19924359023571014 (0.20494904255345328) + | > loss_disc_real_1: 0.24763156473636627 (0.2256257624766014) + | > loss_disc_real_2: 0.24083340167999268 (0.23558148863817283) + | > loss_disc_real_3: 0.300163596868515 (0.2347897710746893) + | > loss_disc_real_4: 0.24460989236831665 (0.23260511302526216) + | > loss_disc_real_5: 0.22363927960395813 (0.2393245167017427) + | > loss_0: 2.7697701454162598 (2.700869450578032) + | > grad_norm_0: tensor(7.7333, device='cuda:0') (tensor(10.5839, device='cuda:0')) + | > loss_gen: 1.8488481044769287 (1.9440179843476364) + | > loss_kl: 2.4676036834716797 (2.5089894646372874) + | > loss_feat: 3.547955274581909 (3.9468391625352854) + | > loss_mel: 19.699573516845703 (20.565865205874893) + | > loss_duration: 1.3962984085083008 (1.3540685791765064) + | > amp_scaler: 256.0 (258.38361266294197) + | > loss_1: 28.960281372070312 (30.319780374594465) + | > grad_norm_1: tensor(216.0488, device='cuda:0') (tensor(190.0219, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 1.2596 (0.924931264678415) + | > loader_time: 0.0205 (0.01465494805874105) + + + --> TIME: 2024-06-25 09:57:21 -- STEP: 562/591 -- GLOBAL_STEP: 25975 + | > loss_disc: 2.7384235858917236 (2.701488946256263) + | > loss_disc_real_0: 0.15795639157295227 (0.2048511934264487) + | > loss_disc_real_1: 0.24698662757873535 (0.2256859028943917) + | > loss_disc_real_2: 0.23517878353595734 (0.23560800345024605) + | > loss_disc_real_3: 0.1930648386478424 (0.23475287581571905) + | > loss_disc_real_4: 0.18970337510108948 (0.23284815638939257) + | > loss_disc_real_5: 0.21240632236003876 (0.23932093000072602) + | > loss_0: 2.7384235858917236 (2.701488946256263) + | > grad_norm_0: tensor(12.3921, device='cuda:0') (tensor(10.4905, device='cuda:0')) + | > loss_gen: 1.9205832481384277 (1.9422779182946555) + | > loss_kl: 2.4674742221832275 (2.5064234139656256) + | > loss_feat: 3.901108980178833 (3.933834880696497) + | > loss_mel: 20.073715209960938 (20.549051359454946) + | > loss_duration: 1.3706750869750977 (1.3556476409324967) + | > amp_scaler: 256.0 (258.2775800711741) + | > loss_1: 29.733558654785156 (30.287235198920307) + | > grad_norm_1: tensor(75.8340, device='cuda:0') (tensor(186.6565, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 1.2151 (0.9373492426719527) + | > loader_time: 0.0218 (0.015041689855772401) + + + --> TIME: 2024-06-25 09:57:53 -- STEP: 587/591 -- GLOBAL_STEP: 26000 + | > loss_disc: 2.726851463317871 (2.701671778648024) + | > loss_disc_real_0: 0.2361515611410141 (0.20477807007768095) + | > loss_disc_real_1: 0.16862128674983978 (0.22573989596744615) + | > loss_disc_real_2: 0.2162516862154007 (0.235655319929326) + | > loss_disc_real_3: 0.2196822166442871 (0.23471770723694635) + | > loss_disc_real_4: 0.22821788489818573 (0.2329330391026843) + | > loss_disc_real_5: 0.26470863819122314 (0.23938870569611734) + | > loss_0: 2.726851463317871 (2.701671778648024) + | > grad_norm_0: tensor(8.7228, device='cuda:0') (tensor(10.5357, device='cuda:0')) + | > loss_gen: 1.9241594076156616 (1.9411059697198298) + | > loss_kl: 2.5356388092041016 (2.505399663363039) + | > loss_feat: 3.6261961460113525 (3.922350308354912) + | > loss_mel: 19.828683853149414 (20.52865673045691) + | > loss_duration: 1.4059584140777588 (1.3573374039472872) + | > amp_scaler: 256.0 (258.180579216354) + | > loss_1: 29.320636749267578 (30.254850065890935) + | > grad_norm_1: tensor(200.2272, device='cuda:0') (tensor(184.0810, device='cuda:0')) + | > current_lr_0: 0.0001989278170603166 + | > current_lr_1: 0.0001989278170603166 + | > step_time: 1.1585 (0.9508467845656801) + | > loader_time: 0.0222 (0.01542203836587259) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003699779510498047 (+3.9136156122735345e-05) + | > avg_loss_disc: 2.8037335720468075 (+0.1340583892578775) + | > avg_loss_disc_real_0: 0.17879098907430122 (+0.011156900131956055) + | > avg_loss_disc_real_1: 0.19267907041184446 (-0.05320164719794654) + | > avg_loss_disc_real_2: 0.24433762754531616 (+0.0012703347713389213) + | > avg_loss_disc_real_3: 0.2696600781476243 (+0.020348269571649147) + | > avg_loss_disc_real_4: 0.2780170954288319 (+0.06282128171717855) + | > avg_loss_disc_real_5: 0.22343290993507872 (+0.008770729316041836) + | > avg_loss_0: 2.8037335720468075 (+0.1340583892578775) + | > avg_loss_gen: 1.7883129855419726 (-0.08653134488044922) + | > avg_loss_kl: 2.6102188850970975 (+0.06733104523192024) + | > avg_loss_feat: 3.7884703443405474 (-0.0550370267096989) + | > avg_loss_mel: 20.536874324717417 (+0.013478299404710015) + | > avg_loss_duration: 1.374277429377779 (-0.005175483987686524) + | > avg_loss_1: 30.09815418973882 (-0.06593448557752168) + + + > EPOCH: 45/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 09:58:05)  + + --> TIME: 2024-06-25 09:58:23 -- STEP: 21/591 -- GLOBAL_STEP: 26025 + | > loss_disc: 2.675306797027588 (2.7049276601700556) + | > loss_disc_real_0: 0.19894525408744812 (0.20414775219701586) + | > loss_disc_real_1: 0.2045196145772934 (0.23546516469546727) + | > loss_disc_real_2: 0.2412772923707962 (0.23858452623798734) + | > loss_disc_real_3: 0.24617277085781097 (0.23043142684868403) + | > loss_disc_real_4: 0.230632022023201 (0.23514807791936965) + | > loss_disc_real_5: 0.21466729044914246 (0.24016563594341278) + | > loss_0: 2.675306797027588 (2.7049276601700556) + | > grad_norm_0: tensor(7.2203, device='cuda:0') (tensor(11.5353, device='cuda:0')) + | > loss_gen: 1.9957925081253052 (1.9661808581579299) + | > loss_kl: 2.6161868572235107 (2.628511803490775) + | > loss_feat: 4.3724589347839355 (4.121336335227603) + | > loss_mel: 21.065940856933594 (20.837391444614955) + | > loss_duration: 1.3393545150756836 (1.3257745334080286) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.389732360839844 (30.879195349557058) + | > grad_norm_1: tensor(217.9133, device='cuda:0') (tensor(240.4700, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8049 (0.7737834680648077) + | > loader_time: 0.0102 (0.008737564086914062) + + + --> TIME: 2024-06-25 09:58:44 -- STEP: 46/591 -- GLOBAL_STEP: 26050 + | > loss_disc: 2.598335027694702 (2.688454633173735) + | > loss_disc_real_0: 0.1280442476272583 (0.20567503522919572) + | > loss_disc_real_1: 0.2042149007320404 (0.2279205808173055) + | > loss_disc_real_2: 0.2193402200937271 (0.23466727655866873) + | > loss_disc_real_3: 0.2586367428302765 (0.23239643288695294) + | > loss_disc_real_4: 0.1710728257894516 (0.22973097828419312) + | > loss_disc_real_5: 0.2202468067407608 (0.24000171539576157) + | > loss_0: 2.598335027694702 (2.688454633173735) + | > grad_norm_0: tensor(12.7804, device='cuda:0') (tensor(11.2159, device='cuda:0')) + | > loss_gen: 2.0397579669952393 (1.9755388031835142) + | > loss_kl: 2.5646042823791504 (2.6024234605872105) + | > loss_feat: 4.5183000564575195 (4.121076226234436) + | > loss_mel: 21.016387939453125 (20.8043735338294) + | > loss_duration: 1.3309502601623535 (1.3314922581548279) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.470001220703125 (30.834904256074324) + | > grad_norm_1: tensor(453.4708, device='cuda:0') (tensor(327.8591, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8101 (0.789407263631406) + | > loader_time: 0.0096 (0.00935096844382908) + + + --> TIME: 2024-06-25 09:59:04 -- STEP: 71/591 -- GLOBAL_STEP: 26075 + | > loss_disc: 2.6961917877197266 (2.6849238335246772) + | > loss_disc_real_0: 0.17084327340126038 (0.20624014283989517) + | > loss_disc_real_1: 0.2615911662578583 (0.2259228611496133) + | > loss_disc_real_2: 0.2194555699825287 (0.236387173264799) + | > loss_disc_real_3: 0.20930910110473633 (0.23299547820024089) + | > loss_disc_real_4: 0.24031154811382294 (0.2272158835975217) + | > loss_disc_real_5: 0.20456643402576447 (0.23785295536820317) + | > loss_0: 2.6961917877197266 (2.6849238335246772) + | > grad_norm_0: tensor(8.6852, device='cuda:0') (tensor(11.8939, device='cuda:0')) + | > loss_gen: 2.060025691986084 (1.9882028992746916) + | > loss_kl: 2.620854377746582 (2.584486447589498) + | > loss_feat: 4.252842426300049 (4.12432140027973) + | > loss_mel: 20.602994918823242 (20.767981193435023) + | > loss_duration: 1.3529462814331055 (1.3373234305583257) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.889663696289062 (30.802315429902414) + | > grad_norm_1: tensor(372.4464, device='cuda:0') (tensor(356.7703, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8259 (0.7982020210212385) + | > loader_time: 0.0101 (0.0096141452520666) + + + --> TIME: 2024-06-25 09:59:26 -- STEP: 96/591 -- GLOBAL_STEP: 26100 + | > loss_disc: 2.7173960208892822 (2.689668245613575) + | > loss_disc_real_0: 0.18018484115600586 (0.2060631668039908) + | > loss_disc_real_1: 0.22663742303848267 (0.22518772026523948) + | > loss_disc_real_2: 0.2166047990322113 (0.23607103091975054) + | > loss_disc_real_3: 0.3253508508205414 (0.2347596261339883) + | > loss_disc_real_4: 0.23323629796504974 (0.22935418986404935) + | > loss_disc_real_5: 0.2591245770454407 (0.23856895385930935) + | > loss_0: 2.7173960208892822 (2.689668245613575) + | > grad_norm_0: tensor(7.6849, device='cuda:0') (tensor(11.8475, device='cuda:0')) + | > loss_gen: 1.7628027200698853 (1.9766027281681697) + | > loss_kl: 2.449704885482788 (2.5653937632838884) + | > loss_feat: 3.841613292694092 (4.096786948541799) + | > loss_mel: 20.032089233398438 (20.747684339682262) + | > loss_duration: 1.3542795181274414 (1.3438401917616527) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.440486907958984 (30.73030795653661) + | > grad_norm_1: tensor(44.6013, device='cuda:0') (tensor(302.2936, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8246 (0.8063870891928673) + | > loader_time: 0.0118 (0.00996966908375422) + + + --> TIME: 2024-06-25 09:59:47 -- STEP: 121/591 -- GLOBAL_STEP: 26125 + | > loss_disc: 2.722837209701538 (2.6919006670802093) + | > loss_disc_real_0: 0.16515403985977173 (0.2050488511762343) + | > loss_disc_real_1: 0.25698646903038025 (0.22562791751928565) + | > loss_disc_real_2: 0.23891761898994446 (0.2360546423876581) + | > loss_disc_real_3: 0.2356637716293335 (0.23453229370196005) + | > loss_disc_real_4: 0.23939940333366394 (0.23026303910026866) + | > loss_disc_real_5: 0.2784525454044342 (0.23888100006363608) + | > loss_0: 2.722837209701538 (2.6919006670802093) + | > grad_norm_0: tensor(17.4446, device='cuda:0') (tensor(11.2869, device='cuda:0')) + | > loss_gen: 1.9455400705337524 (1.9675465506955612) + | > loss_kl: 2.6061246395111084 (2.548976316924923) + | > loss_feat: 3.6954948902130127 (4.073628602934277) + | > loss_mel: 19.841140747070312 (20.71022889633809) + | > loss_duration: 1.3414523601531982 (1.344524803240437) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.429752349853516 (30.64490516914809) + | > grad_norm_1: tensor(301.9601, device='cuda:0') (tensor(279.6979, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8402 (0.814135917947312) + | > loader_time: 0.0118 (0.010259750460790206) + + + --> TIME: 2024-06-25 10:00:09 -- STEP: 146/591 -- GLOBAL_STEP: 26150 + | > loss_disc: 2.65488600730896 (2.688715895561323) + | > loss_disc_real_0: 0.23918500542640686 (0.20463275955352064) + | > loss_disc_real_1: 0.2546989917755127 (0.22552530406272575) + | > loss_disc_real_2: 0.2465093433856964 (0.23532693098260932) + | > loss_disc_real_3: 0.2506888210773468 (0.23434202444471725) + | > loss_disc_real_4: 0.2265935093164444 (0.23011775488314562) + | > loss_disc_real_5: 0.2537912428379059 (0.23854455790699344) + | > loss_0: 2.65488600730896 (2.688715895561323) + | > grad_norm_0: tensor(10.3770, device='cuda:0') (tensor(11.3679, device='cuda:0')) + | > loss_gen: 1.8614957332611084 (1.9684794920764557) + | > loss_kl: 2.5413408279418945 (2.5491197076562333) + | > loss_feat: 3.9380178451538086 (4.077943730027709) + | > loss_mel: 20.132488250732422 (20.692939000586943) + | > loss_duration: 1.3359462022781372 (1.344881554172464) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.809289932250977 (30.633363462474247) + | > grad_norm_1: tensor(163.7514, device='cuda:0') (tensor(287.8747, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8759 (0.8197811172433096) + | > loader_time: 0.0134 (0.010498035443972237) + + + --> TIME: 2024-06-25 10:00:31 -- STEP: 171/591 -- GLOBAL_STEP: 26175 + | > loss_disc: 2.7263693809509277 (2.6891911392323458) + | > loss_disc_real_0: 0.17307212948799133 (0.20441433281926383) + | > loss_disc_real_1: 0.2790239155292511 (0.22550920253260093) + | > loss_disc_real_2: 0.24858583509922028 (0.23565952174844798) + | > loss_disc_real_3: 0.21188154816627502 (0.23400535247130702) + | > loss_disc_real_4: 0.2366143763065338 (0.230115937210663) + | > loss_disc_real_5: 0.21763162314891815 (0.23827466203106773) + | > loss_0: 2.7263693809509277 (2.6891911392323458) + | > grad_norm_0: tensor(8.3642, device='cuda:0') (tensor(11.1760, device='cuda:0')) + | > loss_gen: 2.0268619060516357 (1.966427517216108) + | > loss_kl: 2.4864606857299805 (2.5417765999398036) + | > loss_feat: 3.982477903366089 (4.0790325647209125) + | > loss_mel: 20.66289710998535 (20.681522168611224) + | > loss_duration: 1.3123726844787598 (1.3427719435496641) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.4710693359375 (30.6115307501185) + | > grad_norm_1: tensor(380.1395, device='cuda:0') (tensor(291.4689, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8723 (0.8260575651425367) + | > loader_time: 0.0127 (0.010721305657548513) + + + --> TIME: 2024-06-25 10:00:53 -- STEP: 196/591 -- GLOBAL_STEP: 26200 + | > loss_disc: 2.608966827392578 (2.6862528530918834) + | > loss_disc_real_0: 0.26094990968704224 (0.20386916437015243) + | > loss_disc_real_1: 0.21439358592033386 (0.22533282994919893) + | > loss_disc_real_2: 0.2188107967376709 (0.2353176467728858) + | > loss_disc_real_3: 0.22741690278053284 (0.23411828940924334) + | > loss_disc_real_4: 0.23106566071510315 (0.22999556377834204) + | > loss_disc_real_5: 0.2396259903907776 (0.23826832187419034) + | > loss_0: 2.608966827392578 (2.6862528530918834) + | > grad_norm_0: tensor(14.5762, device='cuda:0') (tensor(11.3199, device='cuda:0')) + | > loss_gen: 1.926289439201355 (1.9673127446855818) + | > loss_kl: 2.4128639698028564 (2.538669055821945) + | > loss_feat: 4.170200824737549 (4.0840597979876465) + | > loss_mel: 20.940874099731445 (20.684416537382166) + | > loss_duration: 1.3581138849258423 (1.3418182925302156) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.80834197998047 (30.61627640042986) + | > grad_norm_1: tensor(292.9193, device='cuda:0') (tensor(293.5340, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8657 (0.8328644499486807) + | > loader_time: 0.0127 (0.010963818248437373) + + + --> TIME: 2024-06-25 10:01:15 -- STEP: 221/591 -- GLOBAL_STEP: 26225 + | > loss_disc: 2.635993719100952 (2.686476921064283) + | > loss_disc_real_0: 0.16723886132240295 (0.2033835743212592) + | > loss_disc_real_1: 0.21782834827899933 (0.22521711953355175) + | > loss_disc_real_2: 0.23466472327709198 (0.23496101493209734) + | > loss_disc_real_3: 0.22274303436279297 (0.23398951134260962) + | > loss_disc_real_4: 0.22051756083965302 (0.23012535617901728) + | > loss_disc_real_5: 0.20535153150558472 (0.23854600938197174) + | > loss_0: 2.635993719100952 (2.686476921064283) + | > grad_norm_0: tensor(6.1073, device='cuda:0') (tensor(11.2284, device='cuda:0')) + | > loss_gen: 1.9862971305847168 (1.9656912719502169) + | > loss_kl: 2.5737011432647705 (2.533679301922139) + | > loss_feat: 4.046079158782959 (4.0799717881560875) + | > loss_mel: 20.904979705810547 (20.68089992644021) + | > loss_duration: 1.3620941638946533 (1.3410934317705319) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.873153686523438 (30.60133567223182) + | > grad_norm_1: tensor(125.8432, device='cuda:0') (tensor(287.6045, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8753 (0.8375469235812917) + | > loader_time: 0.0153 (0.011261661667629602) + + + --> TIME: 2024-06-25 10:01:38 -- STEP: 246/591 -- GLOBAL_STEP: 26250 + | > loss_disc: 2.684324026107788 (2.6860610188507463) + | > loss_disc_real_0: 0.2529509663581848 (0.2031493798746326) + | > loss_disc_real_1: 0.2320093959569931 (0.22523736874989378) + | > loss_disc_real_2: 0.2231694459915161 (0.23503219330213904) + | > loss_disc_real_3: 0.2222171276807785 (0.23397070429916303) + | > loss_disc_real_4: 0.23790480196475983 (0.23045444876197876) + | > loss_disc_real_5: 0.2258012890815735 (0.23856905472230136) + | > loss_0: 2.684324026107788 (2.6860610188507463) + | > grad_norm_0: tensor(8.3857, device='cuda:0') (tensor(10.7664, device='cuda:0')) + | > loss_gen: 1.8413212299346924 (1.9640869948922135) + | > loss_kl: 2.4413681030273438 (2.534528940673767) + | > loss_feat: 3.933601140975952 (4.07694824052051) + | > loss_mel: 20.451059341430664 (20.680932200051906) + | > loss_duration: 1.3591341972351074 (1.3413659930229187) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.026485443115234 (30.59786232894029) + | > grad_norm_1: tensor(97.0313, device='cuda:0') (tensor(274.0573, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8919 (0.8427513120620231) + | > loader_time: 0.0152 (0.011520012607419392) + + + --> TIME: 2024-06-25 10:02:01 -- STEP: 271/591 -- GLOBAL_STEP: 26275 + | > loss_disc: 2.804354667663574 (2.6875860400745357) + | > loss_disc_real_0: 0.41457849740982056 (0.2033671231947262) + | > loss_disc_real_1: 0.17161542177200317 (0.22491453906926723) + | > loss_disc_real_2: 0.21924681961536407 (0.2351846346336097) + | > loss_disc_real_3: 0.23824532330036163 (0.23408803019356464) + | > loss_disc_real_4: 0.21591666340827942 (0.2306846969132054) + | > loss_disc_real_5: 0.1998201161623001 (0.23860267195094556) + | > loss_0: 2.804354667663574 (2.6875860400745357) + | > grad_norm_0: tensor(40.1743, device='cuda:0') (tensor(10.8635, device='cuda:0')) + | > loss_gen: 1.785047173500061 (1.9623494478169403) + | > loss_kl: 2.448579788208008 (2.531283381240395) + | > loss_feat: 3.8869054317474365 (4.069453398679896) + | > loss_mel: 20.357044219970703 (20.654321811295965) + | > loss_duration: 1.35783851146698 (1.3418238897605135) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.83541488647461 (30.559231881285946) + | > grad_norm_1: tensor(195.9698, device='cuda:0') (tensor(267.4969, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.9231 (0.8480978337601102) + | > loader_time: 0.0148 (0.011801098545538981) + + + --> TIME: 2024-06-25 10:02:24 -- STEP: 296/591 -- GLOBAL_STEP: 26300 + | > loss_disc: 2.7089807987213135 (2.689574466363805) + | > loss_disc_real_0: 0.17545630037784576 (0.20363654369941434) + | > loss_disc_real_1: 0.2456621676683426 (0.22525517234729753) + | > loss_disc_real_2: 0.22977642714977264 (0.2349617411760059) + | > loss_disc_real_3: 0.24666418135166168 (0.23431018367409706) + | > loss_disc_real_4: 0.2547592222690582 (0.2310235297961815) + | > loss_disc_real_5: 0.2496071457862854 (0.23877094937740145) + | > loss_0: 2.7089807987213135 (2.689574466363805) + | > grad_norm_0: tensor(7.3078, device='cuda:0') (tensor(10.9491, device='cuda:0')) + | > loss_gen: 1.888692855834961 (1.9606167988197223) + | > loss_kl: 2.6015560626983643 (2.5304654643342306) + | > loss_feat: 3.9702870845794678 (4.058709508663901) + | > loss_mel: 21.18545150756836 (20.656065348032364) + | > loss_duration: 1.3474631309509277 (1.3426835581257535) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.993450164794922 (30.548540637299823) + | > grad_norm_1: tensor(461.7891, device='cuda:0') (tensor(264.1906, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.8982 (0.8534316050039755) + | > loader_time: 0.0164 (0.012042607004578047) + + + --> TIME: 2024-06-25 10:02:48 -- STEP: 321/591 -- GLOBAL_STEP: 26325 + | > loss_disc: 2.68723464012146 (2.6895491155879894) + | > loss_disc_real_0: 0.18046177923679352 (0.20368654540858913) + | > loss_disc_real_1: 0.21574537456035614 (0.22531978112886256) + | > loss_disc_real_2: 0.24191914498806 (0.23493449074271308) + | > loss_disc_real_3: 0.25121650099754333 (0.23442801642826414) + | > loss_disc_real_4: 0.25206315517425537 (0.23083905634293303) + | > loss_disc_real_5: 0.24216262996196747 (0.2386682710822126) + | > loss_0: 2.68723464012146 (2.6895491155879894) + | > grad_norm_0: tensor(7.7918, device='cuda:0') (tensor(10.8267, device='cuda:0')) + | > loss_gen: 1.9776837825775146 (1.9608460081700596) + | > loss_kl: 2.462799310684204 (2.526176350139012) + | > loss_feat: 3.9184300899505615 (4.058566661638635) + | > loss_mel: 20.391773223876953 (20.660250631076906) + | > loss_duration: 1.367537498474121 (1.343309377584131) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.11822509765625 (30.549149005212517) + | > grad_norm_1: tensor(174.7615, device='cuda:0') (tensor(268.0640, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.9709 (0.8588244922435915) + | > loader_time: 0.0163 (0.01231522872069171) + + + --> TIME: 2024-06-25 10:03:12 -- STEP: 346/591 -- GLOBAL_STEP: 26350 + | > loss_disc: 2.7525131702423096 (2.690872787740188) + | > loss_disc_real_0: 0.22159051895141602 (0.20379029173468582) + | > loss_disc_real_1: 0.24271662533283234 (0.22542287297331529) + | > loss_disc_real_2: 0.24933825433254242 (0.23503398533501374) + | > loss_disc_real_3: 0.3376540243625641 (0.23486503954395394) + | > loss_disc_real_4: 0.22835296392440796 (0.23102156572431498) + | > loss_disc_real_5: 0.2218165099620819 (0.23859393161673076) + | > loss_0: 2.7525131702423096 (2.690872787740188) + | > grad_norm_0: tensor(9.9553, device='cuda:0') (tensor(10.8451, device='cuda:0')) + | > loss_gen: 1.82633638381958 (1.9599314666207814) + | > loss_kl: 2.5041189193725586 (2.522646092955088) + | > loss_feat: 3.8273656368255615 (4.050863631887933) + | > loss_mel: 20.597211837768555 (20.652225477847082) + | > loss_duration: 1.380598783493042 (1.344585867975489) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.135629653930664 (30.53025251730329) + | > grad_norm_1: tensor(185.3337, device='cuda:0') (tensor(258.3534, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.9692 (0.8642994523737472) + | > loader_time: 0.0175 (0.01256443930499126) + + + --> TIME: 2024-06-25 10:03:36 -- STEP: 371/591 -- GLOBAL_STEP: 26375 + | > loss_disc: 2.720799207687378 (2.691258328301566) + | > loss_disc_real_0: 0.1524607539176941 (0.2035158950323365) + | > loss_disc_real_1: 0.1883954107761383 (0.22540517460625126) + | > loss_disc_real_2: 0.23646856844425201 (0.23501700765360395) + | > loss_disc_real_3: 0.2502773106098175 (0.2348799302250227) + | > loss_disc_real_4: 0.23495684564113617 (0.23122429514348988) + | > loss_disc_real_5: 0.26688697934150696 (0.2386619015283019) + | > loss_0: 2.720799207687378 (2.691258328301566) + | > grad_norm_0: tensor(16.6183, device='cuda:0') (tensor(10.7571, device='cuda:0')) + | > loss_gen: 2.085308313369751 (1.9587471706847939) + | > loss_kl: 2.4857993125915527 (2.521834761627279) + | > loss_feat: 3.9048895835876465 (4.03960316521781) + | > loss_mel: 20.473155975341797 (20.636584819166508) + | > loss_duration: 1.3738656044006348 (1.3460206940488997) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.32301902770996 (30.502790605282847) + | > grad_norm_1: tensor(61.1722, device='cuda:0') (tensor(258.0105, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.9323 (0.8699524704658118) + | > loader_time: 0.0179 (0.012834548950195307) + + + --> TIME: 2024-06-25 10:04:01 -- STEP: 396/591 -- GLOBAL_STEP: 26400 + | > loss_disc: 2.678088665008545 (2.692950364917216) + | > loss_disc_real_0: 0.23410281538963318 (0.20435662396402676) + | > loss_disc_real_1: 0.22636228799819946 (0.22541275748399772) + | > loss_disc_real_2: 0.26431649923324585 (0.2349069799211892) + | > loss_disc_real_3: 0.24906150996685028 (0.23474229804494165) + | > loss_disc_real_4: 0.22210979461669922 (0.23147447843744298) + | > loss_disc_real_5: 0.23637890815734863 (0.23875994780900503) + | > loss_0: 2.678088665008545 (2.692950364917216) + | > grad_norm_0: tensor(7.3593, device='cuda:0') (tensor(10.9842, device='cuda:0')) + | > loss_gen: 1.8147485256195068 (1.9564320538381132) + | > loss_kl: 2.5064191818237305 (2.5207050966494013) + | > loss_feat: 3.8548200130462646 (4.026928830026375) + | > loss_mel: 20.5811767578125 (20.62622071757462) + | > loss_duration: 1.3854092359542847 (1.3467964406567396) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.142574310302734 (30.477083148378313) + | > grad_norm_1: tensor(281.0437, device='cuda:0') (tensor(248.4347, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 1.0753 (0.8775185605492255) + | > loader_time: 0.0239 (0.01320317959544634) + + + --> TIME: 2024-06-25 10:04:28 -- STEP: 421/591 -- GLOBAL_STEP: 26425 + | > loss_disc: 2.650055408477783 (2.692300905807568) + | > loss_disc_real_0: 0.2522418797016144 (0.20403439360460593) + | > loss_disc_real_1: 0.20228278636932373 (0.22534982350404925) + | > loss_disc_real_2: 0.22189971804618835 (0.23487202292241843) + | > loss_disc_real_3: 0.25042954087257385 (0.23474684343485255) + | > loss_disc_real_4: 0.22752369940280914 (0.2314129213043743) + | > loss_disc_real_5: 0.23730947077274323 (0.23879777503834768) + | > loss_0: 2.650055408477783 (2.692300905807568) + | > grad_norm_0: tensor(12.9019, device='cuda:0') (tensor(10.8658, device='cuda:0')) + | > loss_gen: 1.933851718902588 (1.9553707220775214) + | > loss_kl: 2.5322954654693604 (2.518668583056706) + | > loss_feat: 4.034329414367676 (4.020290420627365) + | > loss_mel: 20.743200302124023 (20.60615689510971) + | > loss_duration: 1.3645811080932617 (1.3476724426423574) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.608257293701172 (30.44815908418415) + | > grad_norm_1: tensor(41.9777, device='cuda:0') (tensor(244.5752, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 0.9824 (0.8874718610577799) + | > loader_time: 0.0183 (0.01379989331804941) + + + --> TIME: 2024-06-25 10:04:54 -- STEP: 446/591 -- GLOBAL_STEP: 26450 + | > loss_disc: 2.737941265106201 (2.6919535884942714) + | > loss_disc_real_0: 0.20209576189517975 (0.2037496512480114) + | > loss_disc_real_1: 0.21088086068630219 (0.22517418119672167) + | > loss_disc_real_2: 0.21759703755378723 (0.23486850350560626) + | > loss_disc_real_3: 0.20445019006729126 (0.23481737534962427) + | > loss_disc_real_4: 0.25212734937667847 (0.23158503166763236) + | > loss_disc_real_5: 0.2674282193183899 (0.23887001622819046) + | > loss_0: 2.737941265106201 (2.6919535884942714) + | > grad_norm_0: tensor(6.6813, device='cuda:0') (tensor(10.7431, device='cuda:0')) + | > loss_gen: 1.9274520874023438 (1.9548651485165138) + | > loss_kl: 2.5202860832214355 (2.5172606578322267) + | > loss_feat: 4.025573253631592 (4.015349853199156) + | > loss_mel: 20.420143127441406 (20.595693314556588) + | > loss_duration: 1.3615059852600098 (1.3489473435376262) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.254961013793945 (30.432116341697796) + | > grad_norm_1: tensor(156.6001, device='cuda:0') (tensor(236.6672, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 1.1381 (0.8951150066649433) + | > loader_time: 0.0179 (0.014079932140127956) + + + --> TIME: 2024-06-25 10:05:22 -- STEP: 471/591 -- GLOBAL_STEP: 26475 + | > loss_disc: 2.7353739738464355 (2.692864370447309) + | > loss_disc_real_0: 0.14179627597332 (0.20355565385643848) + | > loss_disc_real_1: 0.25099101662635803 (0.225327183739648) + | > loss_disc_real_2: 0.21305102109909058 (0.23479405172058476) + | > loss_disc_real_3: 0.21921567618846893 (0.23498307295032875) + | > loss_disc_real_4: 0.22070473432540894 (0.23182118543267502) + | > loss_disc_real_5: 0.2149338722229004 (0.23899206477998422) + | > loss_0: 2.7353739738464355 (2.692864370447309) + | > grad_norm_0: tensor(19.7401, device='cuda:0') (tensor(10.7421, device='cuda:0')) + | > loss_gen: 1.8605422973632812 (1.9523221707394676) + | > loss_kl: 2.455580234527588 (2.515294098297249) + | > loss_feat: 3.6065540313720703 (4.00337823029536) + | > loss_mel: 20.428239822387695 (20.580692611175994) + | > loss_duration: 1.3808026313781738 (1.3501312294836514) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.731718063354492 (30.401818386814902) + | > grad_norm_1: tensor(135.4319, device='cuda:0') (tensor(230.2511, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 1.0932 (0.9042332491297631) + | > loader_time: 0.0188 (0.014487046345024348) + + + --> TIME: 2024-06-25 10:05:49 -- STEP: 496/591 -- GLOBAL_STEP: 26500 + | > loss_disc: 2.656284809112549 (2.6929144873734447) + | > loss_disc_real_0: 0.20172175765037537 (0.20357627810670006) + | > loss_disc_real_1: 0.20929789543151855 (0.2252300291832897) + | > loss_disc_real_2: 0.2287934571504593 (0.23474126315164948) + | > loss_disc_real_3: 0.2147977650165558 (0.23502306937570533) + | > loss_disc_real_4: 0.2341839075088501 (0.23182196689829712) + | > loss_disc_real_5: 0.24817940592765808 (0.23897279579673084) + | > loss_0: 2.656284809112549 (2.6929144873734447) + | > grad_norm_0: tensor(9.3998, device='cuda:0') (tensor(10.7410, device='cuda:0')) + | > loss_gen: 2.0037622451782227 (1.9513260080449042) + | > loss_kl: 2.4756875038146973 (2.513441598703784) + | > loss_feat: 3.935467481613159 (3.995329872735082) + | > loss_mel: 19.910520553588867 (20.564939464292223) + | > loss_duration: 1.3766918182373047 (1.351338777811297) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.70212745666504 (30.376375740574254) + | > grad_norm_1: tensor(152.1112, device='cuda:0') (tensor(227.6138, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 1.1862 (0.9118341385356842) + | > loader_time: 0.0249 (0.01472732184394713) + + + --> TIME: 2024-06-25 10:06:17 -- STEP: 521/591 -- GLOBAL_STEP: 26525 + | > loss_disc: 2.728193521499634 (2.693425222001469) + | > loss_disc_real_0: 0.16079768538475037 (0.20346955767691483) + | > loss_disc_real_1: 0.25074252486228943 (0.22547431281569366) + | > loss_disc_real_2: 0.21329429745674133 (0.23486704206283626) + | > loss_disc_real_3: 0.2390020191669464 (0.234963538035779) + | > loss_disc_real_4: 0.20786677300930023 (0.23187141535149425) + | > loss_disc_real_5: 0.2024931013584137 (0.23897094409662564) + | > loss_0: 2.728193521499634 (2.693425222001469) + | > grad_norm_0: tensor(8.4137, device='cuda:0') (tensor(10.6245, device='cuda:0')) + | > loss_gen: 1.8988196849822998 (1.949686824474591) + | > loss_kl: 2.464385986328125 (2.5133159293323013) + | > loss_feat: 4.211214542388916 (3.988696763245475) + | > loss_mel: 20.83674430847168 (20.55872357783986) + | > loss_duration: 1.3692036867141724 (1.3529296179879426) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.780366897583008 (30.363352733472944) + | > grad_norm_1: tensor(31.2518, device='cuda:0') (tensor(223.4108, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 1.1227 (0.9208760769490775) + | > loader_time: 0.0178 (0.014970855383406213) + + + --> TIME: 2024-06-25 10:06:47 -- STEP: 546/591 -- GLOBAL_STEP: 26550 + | > loss_disc: 2.7225048542022705 (2.6947954500114526) + | > loss_disc_real_0: 0.26923486590385437 (0.2038991149828766) + | > loss_disc_real_1: 0.25889852643013 (0.2256035211158323) + | > loss_disc_real_2: 0.2518796920776367 (0.23486488683647289) + | > loss_disc_real_3: 0.23577064275741577 (0.2348951374491056) + | > loss_disc_real_4: 0.2460150122642517 (0.23200164768067033) + | > loss_disc_real_5: 0.24151940643787384 (0.23907621589663264) + | > loss_0: 2.7225048542022705 (2.6947954500114526) + | > grad_norm_0: tensor(14.5669, device='cuda:0') (tensor(10.7997, device='cuda:0')) + | > loss_gen: 1.861440658569336 (1.9485308255031433) + | > loss_kl: 2.4903154373168945 (2.510473685823517) + | > loss_feat: 3.701781749725342 (3.978814248636962) + | > loss_mel: 20.42148780822754 (20.540906015333242) + | > loss_duration: 1.396092176437378 (1.3545055529136805) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.871116638183594 (30.333230353973725) + | > grad_norm_1: tensor(186.2156, device='cuda:0') (tensor(219.8660, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 1.1264 (0.9320337899001964) + | > loader_time: 0.0187 (0.015245182610256765) + + + --> TIME: 2024-06-25 10:07:17 -- STEP: 571/591 -- GLOBAL_STEP: 26575 + | > loss_disc: 2.7766430377960205 (2.6944708001592743) + | > loss_disc_real_0: 0.2136550396680832 (0.20364028634258832) + | > loss_disc_real_1: 0.2512151300907135 (0.22554270664781279) + | > loss_disc_real_2: 0.24340642988681793 (0.2347433658646201) + | > loss_disc_real_3: 0.2452607899904251 (0.23502102872640573) + | > loss_disc_real_4: 0.25679197907447815 (0.2320269190179487) + | > loss_disc_real_5: 0.2779700458049774 (0.2390578579672999) + | > loss_0: 2.7766430377960205 (2.6944708001592743) + | > grad_norm_0: tensor(6.7499, device='cuda:0') (tensor(10.6960, device='cuda:0')) + | > loss_gen: 1.928553819656372 (1.947325019619302) + | > loss_kl: 2.4240548610687256 (2.5068632337132604) + | > loss_feat: 3.477034091949463 (3.971108506314597) + | > loss_mel: 20.096771240234375 (20.52177353038889) + | > loss_duration: 1.3987715244293213 (1.3562220998904337) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.325185775756836 (30.30329240641953) + | > grad_norm_1: tensor(139.7556, device='cuda:0') (tensor(216.3783, device='cuda:0')) + | > current_lr_0: 0.00019890295108318404 + | > current_lr_1: 0.00019890295108318404 + | > step_time: 1.158 (0.9428944917568601) + | > loader_time: 0.02 (0.015571734533627268) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036882694731367395 (-1.1510037361307356e-05) + | > avg_loss_disc: 2.684192363252032 (-0.11954120879477559) + | > avg_loss_disc_real_0: 0.22586063343159696 (+0.04706964435729574) + | > avg_loss_disc_real_1: 0.2456077717720194 (+0.05292870136017494) + | > avg_loss_disc_real_2: 0.230194923725534 (-0.014142703819782165) + | > avg_loss_disc_real_3: 0.2165417398544068 (-0.05311833829321752) + | > avg_loss_disc_real_4: 0.2447442425058243 (-0.033272852923007606) + | > avg_loss_disc_real_5: 0.22016455264801674 (-0.003268357287061985) + | > avg_loss_0: 2.684192363252032 (-0.11954120879477559) + | > avg_loss_gen: 1.9156533276781122 (+0.12734034213613965) + | > avg_loss_kl: 2.543551627625811 (-0.06666725747128632) + | > avg_loss_feat: 3.8218847842926675 (+0.03341443995212012) + | > avg_loss_mel: 20.31230926513672 (-0.22456505958069783) + | > avg_loss_duration: 1.3759862336706608 (+0.0017088042928816716) + | > avg_loss_1: 29.96938559349547 (-0.12876859624335069) + + + > EPOCH: 46/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 10:07:49)  + + --> TIME: 2024-06-25 10:07:55 -- STEP: 5/591 -- GLOBAL_STEP: 26600 + | > loss_disc: 2.706916332244873 (2.6529232025146485) + | > loss_disc_real_0: 0.14380986988544464 (0.1841227173805237) + | > loss_disc_real_1: 0.217031329870224 (0.22024005949497222) + | > loss_disc_real_2: 0.26130250096321106 (0.23098330199718475) + | > loss_disc_real_3: 0.25892624258995056 (0.23896921277046204) + | > loss_disc_real_4: 0.23016074299812317 (0.2336358278989792) + | > loss_disc_real_5: 0.2428596466779709 (0.24408538043498992) + | > loss_0: 2.706916332244873 (2.6529232025146485) + | > grad_norm_0: tensor(10.3245, device='cuda:0') (tensor(8.0161, device='cuda:0')) + | > loss_gen: 2.103943109512329 (1.996382999420166) + | > loss_kl: 2.6988234519958496 (2.668567419052124) + | > loss_feat: 4.14591646194458 (4.347876644134521) + | > loss_mel: 20.61924171447754 (20.611845779418946) + | > loss_duration: 1.3108060359954834 (1.3356585264205934) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.87873077392578 (30.96033172607422) + | > grad_norm_1: tensor(157.9218, device='cuda:0') (tensor(210.1110, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.78 (0.7607213020324707) + | > loader_time: 0.0082 (0.00812835693359375) + + + --> TIME: 2024-06-25 10:08:15 -- STEP: 30/591 -- GLOBAL_STEP: 26625 + | > loss_disc: 2.699167013168335 (2.6929890473683673) + | > loss_disc_real_0: 0.11984439194202423 (0.20210346529881157) + | > loss_disc_real_1: 0.23080164194107056 (0.22411441802978516) + | > loss_disc_real_2: 0.21921241283416748 (0.23333976318438848) + | > loss_disc_real_3: 0.2317992001771927 (0.2363725225130717) + | > loss_disc_real_4: 0.23753288388252258 (0.23424391746520995) + | > loss_disc_real_5: 0.258911669254303 (0.24023169974486033) + | > loss_0: 2.699167013168335 (2.6929890473683673) + | > grad_norm_0: tensor(16.6360, device='cuda:0') (tensor(11.0584, device='cuda:0')) + | > loss_gen: 2.089944362640381 (1.9624024510383606) + | > loss_kl: 2.4536240100860596 (2.6386166652043666) + | > loss_feat: 4.117151737213135 (4.175709668795268) + | > loss_mel: 20.758684158325195 (20.624622217814125) + | > loss_duration: 1.3373022079467773 (1.3181052724520366) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.75670623779297 (30.719456545511882) + | > grad_norm_1: tensor(99.6542, device='cuda:0') (tensor(140.8527, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.7866 (0.784173854192098) + | > loader_time: 0.0094 (0.009141937891642252) + + + --> TIME: 2024-06-25 10:08:35 -- STEP: 55/591 -- GLOBAL_STEP: 26650 + | > loss_disc: 2.7975399494171143 (2.691758649999445) + | > loss_disc_real_0: 0.2586994767189026 (0.20460858317938715) + | > loss_disc_real_1: 0.25998157262802124 (0.2247362340038473) + | > loss_disc_real_2: 0.25476235151290894 (0.23308002217249438) + | > loss_disc_real_3: 0.30457624793052673 (0.2350782803513787) + | > loss_disc_real_4: 0.27224984765052795 (0.23442219062284989) + | > loss_disc_real_5: 0.26392459869384766 (0.2404657764868303) + | > loss_0: 2.7975399494171143 (2.691758649999445) + | > grad_norm_0: tensor(11.6019, device='cuda:0') (tensor(9.9395, device='cuda:0')) + | > loss_gen: 1.9212493896484375 (1.9613302555951204) + | > loss_kl: 2.5067360401153564 (2.6067632761868564) + | > loss_feat: 3.467893123626709 (4.140178892829199) + | > loss_mel: 19.721925735473633 (20.604539940573954) + | > loss_duration: 1.35577392578125 (1.3289541071111501) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 28.97357940673828 (30.641766530817204) + | > grad_norm_1: tensor(50.5689, device='cuda:0') (tensor(162.0325, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.8138 (0.7950361902063543) + | > loader_time: 0.0097 (0.009593083641745827) + + + --> TIME: 2024-06-25 10:08:56 -- STEP: 80/591 -- GLOBAL_STEP: 26675 + | > loss_disc: 2.669935464859009 (2.691081541776657) + | > loss_disc_real_0: 0.20573672652244568 (0.20465378742665047) + | > loss_disc_real_1: 0.2570948004722595 (0.22448114063590766) + | > loss_disc_real_2: 0.24667061865329742 (0.23312204349786042) + | > loss_disc_real_3: 0.19495320320129395 (0.2344061478972435) + | > loss_disc_real_4: 0.2959747314453125 (0.23346154447644948) + | > loss_disc_real_5: 0.2608281075954437 (0.23945964258164168) + | > loss_0: 2.669935464859009 (2.691081541776657) + | > grad_norm_0: tensor(9.5702, device='cuda:0') (tensor(10.4856, device='cuda:0')) + | > loss_gen: 2.170819044113159 (1.9620175570249558) + | > loss_kl: 2.6054296493530273 (2.5855973303318023) + | > loss_feat: 4.154311180114746 (4.121396216750145) + | > loss_mel: 21.376914978027344 (20.677422428131102) + | > loss_duration: 1.3870102167129517 (1.3359126314520833) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.69448471069336 (30.682346153259278) + | > grad_norm_1: tensor(550.5192, device='cuda:0') (tensor(221.9369, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.8214 (0.8035863757133483) + | > loader_time: 0.0121 (0.009955734014511107) + + + --> TIME: 2024-06-25 10:09:17 -- STEP: 105/591 -- GLOBAL_STEP: 26700 + | > loss_disc: 2.6381866931915283 (2.6885957036699577) + | > loss_disc_real_0: 0.21485230326652527 (0.20378116794994897) + | > loss_disc_real_1: 0.24281761050224304 (0.2258847428219659) + | > loss_disc_real_2: 0.23438112437725067 (0.23381817709831965) + | > loss_disc_real_3: 0.23845969140529633 (0.23455988764762878) + | > loss_disc_real_4: 0.22229480743408203 (0.2314490000406901) + | > loss_disc_real_5: 0.23428986966609955 (0.2394215051616941) + | > loss_0: 2.6381866931915283 (2.6885957036699577) + | > grad_norm_0: tensor(5.3989, device='cuda:0') (tensor(10.4040, device='cuda:0')) + | > loss_gen: 1.8679083585739136 (1.9631908428101312) + | > loss_kl: 2.411700487136841 (2.5631270953587117) + | > loss_feat: 3.9804913997650146 (4.115720921471008) + | > loss_mel: 20.67522430419922 (20.682107325962626) + | > loss_duration: 1.3412877321243286 (1.3412072919663929) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.276613235473633 (30.665353520711264) + | > grad_norm_1: tensor(330.3744, device='cuda:0') (tensor(273.3188, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.8259 (0.8103254795074463) + | > loader_time: 0.0123 (0.010284344355265302) + + + --> TIME: 2024-06-25 10:09:39 -- STEP: 130/591 -- GLOBAL_STEP: 26725 + | > loss_disc: 2.755953073501587 (2.692547948543843) + | > loss_disc_real_0: 0.16723333299160004 (0.20341377413043607) + | > loss_disc_real_1: 0.22252322733402252 (0.22587699798437266) + | > loss_disc_real_2: 0.23681749403476715 (0.2348585212459931) + | > loss_disc_real_3: 0.20024459064006805 (0.23449364419166857) + | > loss_disc_real_4: 0.26617157459259033 (0.2318290209540954) + | > loss_disc_real_5: 0.24480651319026947 (0.23944727606498278) + | > loss_0: 2.755953073501587 (2.692547948543843) + | > grad_norm_0: tensor(17.1877, device='cuda:0') (tensor(10.2889, device='cuda:0')) + | > loss_gen: 2.0691027641296387 (1.9611905033771808) + | > loss_kl: 2.5707366466522217 (2.5568399649399973) + | > loss_feat: 3.843576669692993 (4.101971215468188) + | > loss_mel: 20.39647674560547 (20.68602443108193) + | > loss_duration: 1.3458274602890015 (1.3426798591246971) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.22572135925293 (30.6487060546875) + | > grad_norm_1: tensor(125.7118, device='cuda:0') (tensor(280.1861, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.8427 (0.8157271366852981) + | > loader_time: 0.0115 (0.010563081961411699) + + + --> TIME: 2024-06-25 10:10:01 -- STEP: 155/591 -- GLOBAL_STEP: 26750 + | > loss_disc: 2.6668317317962646 (2.69261548288407) + | > loss_disc_real_0: 0.19633693993091583 (0.20423260807029664) + | > loss_disc_real_1: 0.20468322932720184 (0.22517612749530422) + | > loss_disc_real_2: 0.22774547338485718 (0.23485463876878063) + | > loss_disc_real_3: 0.1769706755876541 (0.23433532945571406) + | > loss_disc_real_4: 0.2314544916152954 (0.2319681901124216) + | > loss_disc_real_5: 0.24763736128807068 (0.2393631470780219) + | > loss_0: 2.6668317317962646 (2.69261548288407) + | > grad_norm_0: tensor(5.4167, device='cuda:0') (tensor(10.5510, device='cuda:0')) + | > loss_gen: 1.9995818138122559 (1.9590948297131445) + | > loss_kl: 2.5324389934539795 (2.5479063557040305) + | > loss_feat: 4.229928970336914 (4.092472337907363) + | > loss_mel: 20.808448791503906 (20.634250911589604) + | > loss_duration: 1.3562633991241455 (1.3416510266642419) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.92666244506836 (30.575375526182114) + | > grad_norm_1: tensor(61.6629, device='cuda:0') (tensor(255.2406, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.8737 (0.8229592554030879) + | > loader_time: 0.0115 (0.01100398340532857) + + + --> TIME: 2024-06-25 10:10:23 -- STEP: 180/591 -- GLOBAL_STEP: 26775 + | > loss_disc: 2.783308506011963 (2.69399488899443) + | > loss_disc_real_0: 0.15471962094306946 (0.20405808368490802) + | > loss_disc_real_1: 0.22054946422576904 (0.22521108604139753) + | > loss_disc_real_2: 0.27573493123054504 (0.23540814220905304) + | > loss_disc_real_3: 0.22956416010856628 (0.23446214016940858) + | > loss_disc_real_4: 0.24770872294902802 (0.23215527443422213) + | > loss_disc_real_5: 0.259623259305954 (0.23933419866694344) + | > loss_0: 2.783308506011963 (2.69399488899443) + | > grad_norm_0: tensor(13.4079, device='cuda:0') (tensor(10.1413, device='cuda:0')) + | > loss_gen: 1.8620374202728271 (1.95673585401641) + | > loss_kl: 2.5524797439575195 (2.5407063696119523) + | > loss_feat: 3.7585506439208984 (4.088360546694864) + | > loss_mel: 20.16797637939453 (20.6288133515252) + | > loss_duration: 1.3175814151763916 (1.3400690184699164) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.658626556396484 (30.554685189988877) + | > grad_norm_1: tensor(63.2998, device='cuda:0') (tensor(232.3964, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.8557 (0.8286065459251404) + | > loader_time: 0.0126 (0.011168550120459673) + + + --> TIME: 2024-06-25 10:10:45 -- STEP: 205/591 -- GLOBAL_STEP: 26800 + | > loss_disc: 2.6659607887268066 (2.693977228025112) + | > loss_disc_real_0: 0.21112188696861267 (0.20372935878794368) + | > loss_disc_real_1: 0.19278953969478607 (0.225410775295118) + | > loss_disc_real_2: 0.23429743945598602 (0.23549178918687308) + | > loss_disc_real_3: 0.26891300082206726 (0.23451151237255188) + | > loss_disc_real_4: 0.26660987734794617 (0.23257474034297757) + | > loss_disc_real_5: 0.2603241205215454 (0.2393843248123076) + | > loss_0: 2.6659607887268066 (2.693977228025112) + | > grad_norm_0: tensor(6.5870, device='cuda:0') (tensor(10.0922, device='cuda:0')) + | > loss_gen: 1.9416379928588867 (1.9554992408287235) + | > loss_kl: 2.5458054542541504 (2.5409574450516126) + | > loss_feat: 4.138416290283203 (4.082736432843094) + | > loss_mel: 20.415103912353516 (20.63085093614532) + | > loss_duration: 1.324783444404602 (1.3393728337636808) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.365745544433594 (30.54941690956674) + | > grad_norm_1: tensor(205.1107, device='cuda:0') (tensor(217.6377, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.8663 (0.8339222663786353) + | > loader_time: 0.0136 (0.0113644425461932) + + + --> TIME: 2024-06-25 10:11:07 -- STEP: 230/591 -- GLOBAL_STEP: 26825 + | > loss_disc: 2.7306463718414307 (2.695459812620413) + | > loss_disc_real_0: 0.12809909880161285 (0.20360262598032536) + | > loss_disc_real_1: 0.18640992045402527 (0.2254788431784381) + | > loss_disc_real_2: 0.2532724142074585 (0.23606737405061723) + | > loss_disc_real_3: 0.2169603705406189 (0.23451678396567055) + | > loss_disc_real_4: 0.2409982681274414 (0.23239997923374175) + | > loss_disc_real_5: 0.2585875988006592 (0.23931230960980707) + | > loss_0: 2.7306463718414307 (2.695459812620413) + | > grad_norm_0: tensor(16.6437, device='cuda:0') (tensor(10.2141, device='cuda:0')) + | > loss_gen: 2.004150867462158 (1.9543507612269857) + | > loss_kl: 2.414743661880493 (2.540418466277745) + | > loss_feat: 4.091292381286621 (4.072976380845775) + | > loss_mel: 20.59918975830078 (20.629902093306832) + | > loss_duration: 1.3544256687164307 (1.3392852327098017) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.46380043029785 (30.536932928665824) + | > grad_norm_1: tensor(238.5544, device='cuda:0') (tensor(209.7742, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.9139 (0.8390962890956712) + | > loader_time: 0.0124 (0.011641537624856708) + + + --> TIME: 2024-06-25 10:11:30 -- STEP: 255/591 -- GLOBAL_STEP: 26850 + | > loss_disc: 2.6844687461853027 (2.6941616507137534) + | > loss_disc_real_0: 0.24275679886341095 (0.20382005353184307) + | > loss_disc_real_1: 0.21141953766345978 (0.22525694013810627) + | > loss_disc_real_2: 0.23629814386367798 (0.23594593791400684) + | > loss_disc_real_3: 0.24397972226142883 (0.23439784634347055) + | > loss_disc_real_4: 0.2290903776884079 (0.232379180485127) + | > loss_disc_real_5: 0.2473021000623703 (0.23929794091804354) + | > loss_0: 2.6844687461853027 (2.6941616507137534) + | > grad_norm_0: tensor(11.9208, device='cuda:0') (tensor(10.0501, device='cuda:0')) + | > loss_gen: 1.7305468320846558 (1.9534766767539231) + | > loss_kl: 2.512644052505493 (2.540163151423137) + | > loss_feat: 3.8789777755737305 (4.070883322697061) + | > loss_mel: 20.93601417541504 (20.634821850645775) + | > loss_duration: 1.314845085144043 (1.3392036634332993) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.373027801513672 (30.53854866027832) + | > grad_norm_1: tensor(247.9123, device='cuda:0') (tensor(214.6246, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.9041 (0.8452849930407954) + | > loader_time: 0.0152 (0.011950821970023368) + + + --> TIME: 2024-06-25 10:11:53 -- STEP: 280/591 -- GLOBAL_STEP: 26875 + | > loss_disc: 2.7091877460479736 (2.692132284811566) + | > loss_disc_real_0: 0.25313788652420044 (0.20380857347377712) + | > loss_disc_real_1: 0.23481221497058868 (0.22540384126561025) + | > loss_disc_real_2: 0.2354666292667389 (0.23578904840563025) + | > loss_disc_real_3: 0.25234586000442505 (0.23437890506216458) + | > loss_disc_real_4: 0.20279720425605774 (0.23155607054276126) + | > loss_disc_real_5: 0.21636910736560822 (0.23901333292680127) + | > loss_0: 2.7091877460479736 (2.692132284811566) + | > grad_norm_0: tensor(15.2464, device='cuda:0') (tensor(10.4216, device='cuda:0')) + | > loss_gen: 1.9466121196746826 (1.9596553002085004) + | > loss_kl: 2.5958011150360107 (2.538514656679972) + | > loss_feat: 4.111242771148682 (4.0802658779280545) + | > loss_mel: 20.777267456054688 (20.635856989451813) + | > loss_duration: 1.3674302101135254 (1.3398668485028404) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.798351287841797 (30.554159695761545) + | > grad_norm_1: tensor(170.8732, device='cuda:0') (tensor(235.0460, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.9054 (0.8501488489764077) + | > loader_time: 0.0143 (0.012172749212809978) + + + --> TIME: 2024-06-25 10:12:17 -- STEP: 305/591 -- GLOBAL_STEP: 26900 + | > loss_disc: 2.6692545413970947 (2.6922535184954044) + | > loss_disc_real_0: 0.25779759883880615 (0.20362471408042757) + | > loss_disc_real_1: 0.27036672830581665 (0.2254638618133107) + | > loss_disc_real_2: 0.2589929401874542 (0.23589328752189387) + | > loss_disc_real_3: 0.23967275023460388 (0.23430808855861915) + | > loss_disc_real_4: 0.23440532386302948 (0.2317387749914263) + | > loss_disc_real_5: 0.24326331913471222 (0.23900100007408956) + | > loss_0: 2.6692545413970947 (2.6922535184954044) + | > grad_norm_0: tensor(9.7681, device='cuda:0') (tensor(10.4088, device='cuda:0')) + | > loss_gen: 1.842807650566101 (1.959227318841903) + | > loss_kl: 2.582916736602783 (2.541086525995226) + | > loss_feat: 3.8530466556549072 (4.073942304048384) + | > loss_mel: 20.84075164794922 (20.636992839125334) + | > loss_duration: 1.328782558441162 (1.3406566858291629) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.448305130004883 (30.551905703935468) + | > grad_norm_1: tensor(145.5046, device='cuda:0') (tensor(239.5238, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.9101 (0.855466094564219) + | > loader_time: 0.0156 (0.012459855783181116) + + + --> TIME: 2024-06-25 10:12:40 -- STEP: 330/591 -- GLOBAL_STEP: 26925 + | > loss_disc: 2.6397900581359863 (2.6923485510277043) + | > loss_disc_real_0: 0.12827260792255402 (0.20349040514591968) + | > loss_disc_real_1: 0.2282811999320984 (0.22529162642630665) + | > loss_disc_real_2: 0.24279677867889404 (0.23586192013639393) + | > loss_disc_real_3: 0.26281067728996277 (0.23432326059449804) + | > loss_disc_real_4: 0.2219374179840088 (0.2316502504276507) + | > loss_disc_real_5: 0.21782000362873077 (0.2388792240258419) + | > loss_0: 2.6397900581359863 (2.6923485510277043) + | > grad_norm_0: tensor(15.3097, device='cuda:0') (tensor(10.3441, device='cuda:0')) + | > loss_gen: 2.116941452026367 (1.9586552973949547) + | > loss_kl: 2.5056846141815186 (2.536755870327808) + | > loss_feat: 4.1153106689453125 (4.067597933971524) + | > loss_mel: 20.373004913330078 (20.634748447302616) + | > loss_duration: 1.352675437927246 (1.341742990594922) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.46361541748047 (30.53950054284298) + | > grad_norm_1: tensor(300.9604, device='cuda:0') (tensor(235.1854, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.9202 (0.8608054623459325) + | > loader_time: 0.0172 (0.012682319409919512) + + + --> TIME: 2024-06-25 10:13:04 -- STEP: 355/591 -- GLOBAL_STEP: 26950 + | > loss_disc: 2.714601755142212 (2.6927395941505985) + | > loss_disc_real_0: 0.22088277339935303 (0.20389577985649376) + | > loss_disc_real_1: 0.23597508668899536 (0.2251405183278339) + | > loss_disc_real_2: 0.23045817017555237 (0.2357853907514626) + | > loss_disc_real_3: 0.1963246613740921 (0.23415810768033418) + | > loss_disc_real_4: 0.2355974316596985 (0.2318939340366444) + | > loss_disc_real_5: 0.2757651209831238 (0.23902338004448045) + | > loss_0: 2.714601755142212 (2.6927395941505985) + | > grad_norm_0: tensor(4.8988, device='cuda:0') (tensor(10.2507, device='cuda:0')) + | > loss_gen: 1.9209706783294678 (1.956920976705954) + | > loss_kl: 2.5226504802703857 (2.5340086742186227) + | > loss_feat: 3.779844045639038 (4.05479201733227) + | > loss_mel: 20.083335876464844 (20.624342125906075) + | > loss_duration: 1.3382080793380737 (1.3430918535716105) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.645008087158203 (30.513155687359017) + | > grad_norm_1: tensor(54.4619, device='cuda:0') (tensor(230.3449, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.9262 (0.8662534485400563) + | > loader_time: 0.0151 (0.012922149980571913) + + + --> TIME: 2024-06-25 10:13:29 -- STEP: 380/591 -- GLOBAL_STEP: 26975 + | > loss_disc: 2.661738872528076 (2.692143210611846) + | > loss_disc_real_0: 0.21679513156414032 (0.20356567641230006) + | > loss_disc_real_1: 0.23544244468212128 (0.2250595133555563) + | > loss_disc_real_2: 0.24687010049819946 (0.2356469485712679) + | > loss_disc_real_3: 0.2390407770872116 (0.23425034653199348) + | > loss_disc_real_4: 0.2314605861902237 (0.23202996842171017) + | > loss_disc_real_5: 0.25233837962150574 (0.23902382203622868) + | > loss_0: 2.661738872528076 (2.692143210611846) + | > grad_norm_0: tensor(5.9024, device='cuda:0') (tensor(10.2557, device='cuda:0')) + | > loss_gen: 1.9230694770812988 (1.9557674166403318) + | > loss_kl: 2.4250478744506836 (2.5329018705769606) + | > loss_feat: 3.981658458709717 (4.047154092788699) + | > loss_mel: 20.202415466308594 (20.604826962320434) + | > loss_duration: 1.331605076789856 (1.344640054200826) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.86379623413086 (30.48529042193764) + | > grad_norm_1: tensor(212.6833, device='cuda:0') (tensor(225.8141, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.9661 (0.8718529174202367) + | > loader_time: 0.0185 (0.013180703865854368) + + + --> TIME: 2024-06-25 10:13:54 -- STEP: 405/591 -- GLOBAL_STEP: 27000 + | > loss_disc: 2.733574867248535 (2.6920775272228106) + | > loss_disc_real_0: 0.26318997144699097 (0.2033463497036769) + | > loss_disc_real_1: 0.23671968281269073 (0.22506526874171362) + | > loss_disc_real_2: 0.25643402338027954 (0.23560401289551347) + | > loss_disc_real_3: 0.24201075732707977 (0.23438660623850646) + | > loss_disc_real_4: 0.25633570551872253 (0.23221046961384056) + | > loss_disc_real_5: 0.2625862956047058 (0.2390101233014354) + | > loss_0: 2.733574867248535 (2.6920775272228106) + | > grad_norm_0: tensor(10.4925, device='cuda:0') (tensor(10.1266, device='cuda:0')) + | > loss_gen: 1.8142621517181396 (1.954450246139809) + | > loss_kl: 2.489208936691284 (2.5303971549611055) + | > loss_feat: 3.7465438842773438 (4.039376320073638) + | > loss_mel: 20.145071029663086 (20.600813740859795) + | > loss_duration: 1.3397421836853027 (1.3457427890212459) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.53483009338379 (30.470780276074823) + | > grad_norm_1: tensor(169.3163, device='cuda:0') (tensor(225.8060, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 0.997 (0.877963583557694) + | > loader_time: 0.0185 (0.013470285910147211) + + + --> TIME: 2024-06-25 10:14:19 -- STEP: 430/591 -- GLOBAL_STEP: 27025 + | > loss_disc: 2.6470210552215576 (2.6925092020700148) + | > loss_disc_real_0: 0.22104671597480774 (0.20362424963088924) + | > loss_disc_real_1: 0.2306956648826599 (0.22505300277194312) + | > loss_disc_real_2: 0.23738129436969757 (0.235493173086366) + | > loss_disc_real_3: 0.27357712388038635 (0.23438160239264022) + | > loss_disc_real_4: 0.23777393996715546 (0.23209917628487875) + | > loss_disc_real_5: 0.2472509890794754 (0.23900732820810275) + | > loss_0: 2.6470210552215576 (2.6925092020700148) + | > grad_norm_0: tensor(4.6458, device='cuda:0') (tensor(10.3004, device='cuda:0')) + | > loss_gen: 1.8790525197982788 (1.953748607358267) + | > loss_kl: 2.4969446659088135 (2.529610274004384) + | > loss_feat: 3.8745861053466797 (4.031665724377305) + | > loss_mel: 19.86592674255371 (20.584430951850358) + | > loss_duration: 1.3431406021118164 (1.3464605323103982) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.459651947021484 (30.445916127049646) + | > grad_norm_1: tensor(144.1621, device='cuda:0') (tensor(222.7413, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 1.0088 (0.884247998858607) + | > loader_time: 0.0197 (0.013757872027020127) + + + --> TIME: 2024-06-25 10:14:45 -- STEP: 455/591 -- GLOBAL_STEP: 27050 + | > loss_disc: 2.7113571166992188 (2.6927062270405533) + | > loss_disc_real_0: 0.18256452679634094 (0.20362575905663627) + | > loss_disc_real_1: 0.24642136693000793 (0.22506051066812577) + | > loss_disc_real_2: 0.2555588185787201 (0.23563238971835965) + | > loss_disc_real_3: 0.21589817106723785 (0.23431611974815747) + | > loss_disc_real_4: 0.22955185174942017 (0.23212613073679117) + | > loss_disc_real_5: 0.23833219707012177 (0.23898458294161073) + | > loss_0: 2.7113571166992188 (2.6927062270405533) + | > grad_norm_0: tensor(6.8698, device='cuda:0') (tensor(10.3447, device='cuda:0')) + | > loss_gen: 1.880883812904358 (1.9528554271865677) + | > loss_kl: 2.5887022018432617 (2.527659977923385) + | > loss_feat: 3.990144968032837 (4.0211451630016) + | > loss_mel: 20.417179107666016 (20.567142775818542) + | > loss_duration: 1.3661694526672363 (1.3475770735478674) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.243078231811523 (30.416380452585745) + | > grad_norm_1: tensor(249.9105, device='cuda:0') (tensor(218.8295, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 1.006 (0.8912454143985287) + | > loader_time: 0.0185 (0.014023212810139083) + + + --> TIME: 2024-06-25 10:15:11 -- STEP: 480/591 -- GLOBAL_STEP: 27075 + | > loss_disc: 2.666548013687134 (2.6944526895880703) + | > loss_disc_real_0: 0.28729045391082764 (0.2043025551829487) + | > loss_disc_real_1: 0.22494572401046753 (0.22514827499787013) + | > loss_disc_real_2: 0.24497286975383759 (0.23558405910929045) + | > loss_disc_real_3: 0.24801971018314362 (0.23438071440905334) + | > loss_disc_real_4: 0.2500756084918976 (0.23220414600024622) + | > loss_disc_real_5: 0.24279701709747314 (0.23898830168570082) + | > loss_0: 2.666548013687134 (2.6944526895880703) + | > grad_norm_0: tensor(16.3834, device='cuda:0') (tensor(10.5529, device='cuda:0')) + | > loss_gen: 1.9929759502410889 (1.9520518243312837) + | > loss_kl: 2.587660789489746 (2.5266949037710846) + | > loss_feat: 3.7260818481445312 (4.011120110750204) + | > loss_mel: 20.37767791748047 (20.555832660198213) + | > loss_duration: 1.3676398992538452 (1.3488430067896853) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.05203628540039 (30.394542531172434) + | > grad_norm_1: tensor(66.9921, device='cuda:0') (tensor(214.3971, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 1.0965 (0.8984129667282105) + | > loader_time: 0.0258 (0.014286170403162641) + + + --> TIME: 2024-06-25 10:15:39 -- STEP: 505/591 -- GLOBAL_STEP: 27100 + | > loss_disc: 2.6322803497314453 (2.6944362069120507) + | > loss_disc_real_0: 0.21727973222732544 (0.20393522563252117) + | > loss_disc_real_1: 0.25090181827545166 (0.2251875910428491) + | > loss_disc_real_2: 0.20163869857788086 (0.23561940514805294) + | > loss_disc_real_3: 0.2231704741716385 (0.23440906945431586) + | > loss_disc_real_4: 0.2354530394077301 (0.23224334864333124) + | > loss_disc_real_5: 0.2365749180316925 (0.23905406429035828) + | > loss_0: 2.6322803497314453 (2.6944362069120507) + | > grad_norm_0: tensor(7.9255, device='cuda:0') (tensor(10.3642, device='cuda:0')) + | > loss_gen: 1.9672019481658936 (1.9506717679524186) + | > loss_kl: 2.429002046585083 (2.524331143823002) + | > loss_feat: 4.13338565826416 (4.001876931143283) + | > loss_mel: 20.505823135375977 (20.538928985595703) + | > loss_duration: 1.379054069519043 (1.3502352063018506) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.414466857910156 (30.366044069044662) + | > grad_norm_1: tensor(315.5003, device='cuda:0') (tensor(212.5814, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 1.0557 (0.9074331812339254) + | > loader_time: 0.0173 (0.014538741819929372) + + + --> TIME: 2024-06-25 10:16:08 -- STEP: 530/591 -- GLOBAL_STEP: 27125 + | > loss_disc: 2.7863149642944336 (2.6953658954152524) + | > loss_disc_real_0: 0.30811554193496704 (0.20385385469164488) + | > loss_disc_real_1: 0.2164192497730255 (0.2253034624850975) + | > loss_disc_real_2: 0.27550235390663147 (0.2357681944685162) + | > loss_disc_real_3: 0.2603318691253662 (0.23461992687211847) + | > loss_disc_real_4: 0.22829921543598175 (0.23219151204487062) + | > loss_disc_real_5: 0.2402595579624176 (0.23906562657288785) + | > loss_0: 2.7863149642944336 (2.6953658954152524) + | > grad_norm_0: tensor(19.0086, device='cuda:0') (tensor(10.3049, device='cuda:0')) + | > loss_gen: 1.7416645288467407 (1.9492118084205772) + | > loss_kl: 2.528987169265747 (2.523004733841376) + | > loss_feat: 3.72524356842041 (3.99166907454437) + | > loss_mel: 20.295534133911133 (20.52571051975466) + | > loss_duration: 1.3827590942382812 (1.3518502710000533) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.6741886138916 (30.3414464518709) + | > grad_norm_1: tensor(116.6938, device='cuda:0') (tensor(214.4865, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 1.2235 (0.9183395930056302) + | > loader_time: 0.0203 (0.01481565079599057) + + + --> TIME: 2024-06-25 10:16:39 -- STEP: 555/591 -- GLOBAL_STEP: 27150 + | > loss_disc: 2.6324291229248047 (2.696760314649291) + | > loss_disc_real_0: 0.21013081073760986 (0.20412506705468839) + | > loss_disc_real_1: 0.25076785683631897 (0.22533392060447383) + | > loss_disc_real_2: 0.24239470064640045 (0.23577950728369187) + | > loss_disc_real_3: 0.2365962564945221 (0.2347674158510861) + | > loss_disc_real_4: 0.21512135863304138 (0.23221240908175975) + | > loss_disc_real_5: 0.22936585545539856 (0.2390493405294848) + | > loss_0: 2.6324291229248047 (2.696760314649291) + | > grad_norm_0: tensor(6.9129, device='cuda:0') (tensor(10.3008, device='cuda:0')) + | > loss_gen: 1.9879056215286255 (1.9483139233546214) + | > loss_kl: 2.510270833969116 (2.5208401624146894) + | > loss_feat: 3.981494903564453 (3.9809797261212356) + | > loss_mel: 20.257179260253906 (20.51424571544201) + | > loss_duration: 1.4230425357818604 (1.3535937474654607) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.159893035888672 (30.317973307016732) + | > grad_norm_1: tensor(512.1845, device='cuda:0') (tensor(211.5206, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 1.1886 (0.9315356765781437) + | > loader_time: 0.0221 (0.015126636006810643) + + + --> TIME: 2024-06-25 10:17:11 -- STEP: 580/591 -- GLOBAL_STEP: 27175 + | > loss_disc: 2.659139633178711 (2.6966704627563227) + | > loss_disc_real_0: 0.0973842442035675 (0.20422427231638599) + | > loss_disc_real_1: 0.21950402855873108 (0.225153507266579) + | > loss_disc_real_2: 0.2323862463235855 (0.23582592971366026) + | > loss_disc_real_3: 0.24760861694812775 (0.2347963242438333) + | > loss_disc_real_4: 0.2124047875404358 (0.23205619534027988) + | > loss_disc_real_5: 0.2524661123752594 (0.2391505637816314) + | > loss_0: 2.659139633178711 (2.6966704627563227) + | > grad_norm_0: tensor(27.3017, device='cuda:0') (tensor(10.3990, device='cuda:0')) + | > loss_gen: 2.1562063694000244 (1.9487736541649392) + | > loss_kl: 2.509404182434082 (2.5191002545685612) + | > loss_feat: 3.853736400604248 (3.9749406202086117) + | > loss_mel: 20.16047477722168 (20.501530364464074) + | > loss_duration: 1.3925975561141968 (1.3553203745134947) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.072418212890625 (30.299665299777327) + | > grad_norm_1: tensor(623.1137, device='cuda:0') (tensor(217.7231, device='cuda:0')) + | > current_lr_0: 0.00019887808821429862 + | > current_lr_1: 0.00019887808821429862 + | > step_time: 1.2993 (0.945785156200672) + | > loader_time: 0.029 (0.015465375883825894) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037802381718412357 (+9.196869870449617e-05) + | > avg_loss_disc: 2.665613742584878 (-0.018578620667153878) + | > avg_loss_disc_real_0: 0.2965965933622198 (+0.07073595993062282) + | > avg_loss_disc_real_1: 0.2205515700451871 (-0.02505620172683229) + | > avg_loss_disc_real_2: 0.22510120494568603 (-0.005093718779847961) + | > avg_loss_disc_real_3: 0.1980435626303896 (-0.018498177224017198) + | > avg_loss_disc_real_4: 0.24268677069785746 (-0.0020574718079668375) + | > avg_loss_disc_real_5: 0.23013006975042058 (+0.009965517102403842) + | > avg_loss_0: 2.665613742584878 (-0.018578620667153878) + | > avg_loss_gen: 2.04244098511148 (+0.12678765743336773) + | > avg_loss_kl: 2.5696196657546024 (+0.026068038128791216) + | > avg_loss_feat: 3.982744993047513 (+0.1608602087548454) + | > avg_loss_mel: 20.805188118143285 (+0.492878853006566) + | > avg_loss_duration: 1.3704709144348795 (-0.005515319235781302) + | > avg_loss_1: 30.77046467395539 (+0.801079080459921) + + + > EPOCH: 47/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 10:17:32)  + + --> TIME: 2024-06-25 10:17:45 -- STEP: 14/591 -- GLOBAL_STEP: 27200 + | > loss_disc: 2.721348285675049 (2.6453610488346646) + | > loss_disc_real_0: 0.22189980745315552 (0.1979540256517274) + | > loss_disc_real_1: 0.30108779668807983 (0.22582068400723593) + | > loss_disc_real_2: 0.2399124950170517 (0.23417552879878453) + | > loss_disc_real_3: 0.23241207003593445 (0.2374079472252301) + | > loss_disc_real_4: 0.2907998561859131 (0.22081803530454636) + | > loss_disc_real_5: 0.23286493122577667 (0.23565878186907088) + | > loss_0: 2.721348285675049 (2.6453610488346646) + | > grad_norm_0: tensor(5.8023, device='cuda:0') (tensor(8.7198, device='cuda:0')) + | > loss_gen: 1.9652249813079834 (2.0542496187346324) + | > loss_kl: 2.6915035247802734 (2.6912133182798113) + | > loss_feat: 4.186532497406006 (4.368889399937221) + | > loss_mel: 20.414356231689453 (20.829560007367814) + | > loss_duration: 1.306573510169983 (1.3166725465229578) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.564191818237305 (31.26058510371617) + | > grad_norm_1: tensor(318.1460, device='cuda:0') (tensor(470.1176, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.7658 (0.7726612261363438) + | > loader_time: 0.0087 (0.00899214403969901) + + + --> TIME: 2024-06-25 10:18:05 -- STEP: 39/591 -- GLOBAL_STEP: 27225 + | > loss_disc: 2.671624183654785 (2.6667438470400295) + | > loss_disc_real_0: 0.1509379744529724 (0.19978198714745352) + | > loss_disc_real_1: 0.17673321068286896 (0.2219934723316095) + | > loss_disc_real_2: 0.22037576138973236 (0.2314625050012882) + | > loss_disc_real_3: 0.21668073534965515 (0.23470247173920655) + | > loss_disc_real_4: 0.24881525337696075 (0.22740220641478515) + | > loss_disc_real_5: 0.2488579899072647 (0.23790835035152924) + | > loss_0: 2.671624183654785 (2.6667438470400295) + | > grad_norm_0: tensor(10.3570, device='cuda:0') (tensor(8.2641, device='cuda:0')) + | > loss_gen: 2.215993642807007 (2.0023961831361823) + | > loss_kl: 2.6871907711029053 (2.6460185601161075) + | > loss_feat: 4.419553756713867 (4.241422683764726) + | > loss_mel: 20.964962005615234 (20.648629212990784) + | > loss_duration: 1.3399240970611572 (1.3221232524284947) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.62762451171875 (30.8605899810791) + | > grad_norm_1: tensor(311.1876, device='cuda:0') (tensor(372.2575, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.8198 (0.787255904613397) + | > loader_time: 0.0105 (0.009456438895983573) + + + --> TIME: 2024-06-25 10:18:26 -- STEP: 64/591 -- GLOBAL_STEP: 27250 + | > loss_disc: 2.666203498840332 (2.675246313214301) + | > loss_disc_real_0: 0.1688913106918335 (0.201488449703902) + | > loss_disc_real_1: 0.24572260677814484 (0.22248144843615592) + | > loss_disc_real_2: 0.2517383098602295 (0.23257723404094577) + | > loss_disc_real_3: 0.25354012846946716 (0.2343235716689378) + | > loss_disc_real_4: 0.25020286440849304 (0.2306082881987095) + | > loss_disc_real_5: 0.26218464970588684 (0.23961191531270742) + | > loss_0: 2.666203498840332 (2.675246313214301) + | > grad_norm_0: tensor(9.2889, device='cuda:0') (tensor(9.3746, device='cuda:0')) + | > loss_gen: 1.8843610286712646 (1.9731047637760641) + | > loss_kl: 2.509899139404297 (2.5956787914037704) + | > loss_feat: 3.852627992630005 (4.173663932830095) + | > loss_mel: 20.26251792907715 (20.60184994339943) + | > loss_duration: 1.351917028427124 (1.3315866719931364) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.861324310302734 (30.675884127616882) + | > grad_norm_1: tensor(141.6604, device='cuda:0') (tensor(295.7389, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.8227 (0.7963174916803837) + | > loader_time: 0.0107 (0.009735986590385437) + + + --> TIME: 2024-06-25 10:18:47 -- STEP: 89/591 -- GLOBAL_STEP: 27275 + | > loss_disc: 2.691472053527832 (2.674640963586528) + | > loss_disc_real_0: 0.1699746549129486 (0.20161438850539454) + | > loss_disc_real_1: 0.2674064636230469 (0.22258871308203493) + | > loss_disc_real_2: 0.2563183307647705 (0.23244949086998287) + | > loss_disc_real_3: 0.27399981021881104 (0.23376385415537973) + | > loss_disc_real_4: 0.2387433797121048 (0.23062116795041587) + | > loss_disc_real_5: 0.2389955222606659 (0.23912656139791683) + | > loss_0: 2.691472053527832 (2.674640963586528) + | > grad_norm_0: tensor(9.0255, device='cuda:0') (tensor(9.9778, device='cuda:0')) + | > loss_gen: 2.020380735397339 (1.9749619759870378) + | > loss_kl: 2.353900671005249 (2.5749669316109656) + | > loss_feat: 4.152204990386963 (4.172648234313793) + | > loss_mel: 21.000585556030273 (20.618045271112674) + | > loss_duration: 1.3829257488250732 (1.3382029372654607) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.909996032714844 (30.678825292694434) + | > grad_norm_1: tensor(366.4548, device='cuda:0') (tensor(285.5293, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.8206 (0.8045387375220824) + | > loader_time: 0.0105 (0.010125548652048864) + + + --> TIME: 2024-06-25 10:19:08 -- STEP: 114/591 -- GLOBAL_STEP: 27300 + | > loss_disc: 2.66058087348938 (2.673840915947629) + | > loss_disc_real_0: 0.28240954875946045 (0.201178410270235) + | > loss_disc_real_1: 0.27046194672584534 (0.22345619792478127) + | > loss_disc_real_2: 0.2304447889328003 (0.2324426533621654) + | > loss_disc_real_3: 0.23458103835582733 (0.23332265882115616) + | > loss_disc_real_4: 0.20676691830158234 (0.230271851545886) + | > loss_disc_real_5: 0.22171266376972198 (0.2393414324574303) + | > loss_0: 2.66058087348938 (2.673840915947629) + | > grad_norm_0: tensor(21.1794, device='cuda:0') (tensor(9.7154, device='cuda:0')) + | > loss_gen: 1.8373948335647583 (1.973701366207056) + | > loss_kl: 2.464838981628418 (2.562278741284421) + | > loss_feat: 3.938337564468384 (4.172883169692861) + | > loss_mel: 20.042110443115234 (20.638530982168096) + | > loss_duration: 1.3733837604522705 (1.3411737826832555) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.656064987182617 (30.688568031578733) + | > grad_norm_1: tensor(55.4742, device='cuda:0') (tensor(296.7036, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.8332 (0.8113163500501398) + | > loader_time: 0.0123 (0.010466333021197406) + + + --> TIME: 2024-06-25 10:19:30 -- STEP: 139/591 -- GLOBAL_STEP: 27325 + | > loss_disc: 2.7171599864959717 (2.675652279270638) + | > loss_disc_real_0: 0.275505006313324 (0.2016721397316713) + | > loss_disc_real_1: 0.2326197624206543 (0.22355006142057104) + | > loss_disc_real_2: 0.22812287509441376 (0.2322989219813038) + | > loss_disc_real_3: 0.23848353326320648 (0.23372114412218548) + | > loss_disc_real_4: 0.21504351496696472 (0.2300192186300703) + | > loss_disc_real_5: 0.23747406899929047 (0.23899231400850007) + | > loss_0: 2.7171599864959717 (2.675652279270638) + | > grad_norm_0: tensor(14.8634, device='cuda:0') (tensor(10.1715, device='cuda:0')) + | > loss_gen: 1.8729567527770996 (1.9726045371817171) + | > loss_kl: 2.618483066558838 (2.5609526565606644) + | > loss_feat: 3.948774576187134 (4.164644294505498) + | > loss_mel: 20.74821662902832 (20.635528934945306) + | > loss_duration: 1.360651969909668 (1.3422462168357354) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.549083709716797 (30.675976664042302) + | > grad_norm_1: tensor(197.2575, device='cuda:0') (tensor(305.7157, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.8597 (0.8172665757241009) + | > loader_time: 0.011 (0.010676516045769345) + + + --> TIME: 2024-06-25 10:19:51 -- STEP: 164/591 -- GLOBAL_STEP: 27350 + | > loss_disc: 2.6342761516571045 (2.6754702475012793) + | > loss_disc_real_0: 0.2440212219953537 (0.20096926404753837) + | > loss_disc_real_1: 0.23434585332870483 (0.22347079026626376) + | > loss_disc_real_2: 0.19973339140415192 (0.23301203394444978) + | > loss_disc_real_3: 0.20035004615783691 (0.23407068966728886) + | > loss_disc_real_4: 0.24600131809711456 (0.2302472474189793) + | > loss_disc_real_5: 0.25943848490715027 (0.23909008802800644) + | > loss_0: 2.6342761516571045 (2.6754702475012793) + | > grad_norm_0: tensor(16.7183, device='cuda:0') (tensor(10.1158, device='cuda:0')) + | > loss_gen: 1.8963418006896973 (1.9734807755888961) + | > loss_kl: 2.664616584777832 (2.5566510529052913) + | > loss_feat: 3.999074697494507 (4.164223301701432) + | > loss_mel: 20.190793991088867 (20.644568233955198) + | > loss_duration: 1.3487735986709595 (1.3405334273489504) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.099599838256836 (30.679456815487) + | > grad_norm_1: tensor(61.8413, device='cuda:0') (tensor(311.6192, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.8578 (0.8234189094566717) + | > loader_time: 0.0116 (0.01094529396150171) + + + --> TIME: 2024-06-25 10:20:14 -- STEP: 189/591 -- GLOBAL_STEP: 27375 + | > loss_disc: 2.694226026535034 (2.678424857911609) + | > loss_disc_real_0: 0.18243622779846191 (0.20135593887359376) + | > loss_disc_real_1: 0.20741619169712067 (0.22365454328123224) + | > loss_disc_real_2: 0.21912093460559845 (0.23326880693751037) + | > loss_disc_real_3: 0.23234155774116516 (0.2343316168066055) + | > loss_disc_real_4: 0.2221972644329071 (0.2301898746105729) + | > loss_disc_real_5: 0.22553651034832 (0.23900961710347068) + | > loss_0: 2.694226026535034 (2.678424857911609) + | > grad_norm_0: tensor(5.4621, device='cuda:0') (tensor(10.4069, device='cuda:0')) + | > loss_gen: 1.964033603668213 (1.9719173769471507) + | > loss_kl: 2.627654552459717 (2.553014859951363) + | > loss_feat: 4.098506450653076 (4.147617668071123) + | > loss_mel: 20.26127052307129 (20.63485830801504) + | > loss_duration: 1.322322964668274 (1.3387228401880409) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.273786544799805 (30.64613104118872) + | > grad_norm_1: tensor(87.0175, device='cuda:0') (tensor(302.7343, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.8548 (0.8289111480511054) + | > loader_time: 0.0124 (0.011167392528877065) + + + --> TIME: 2024-06-25 10:20:36 -- STEP: 214/591 -- GLOBAL_STEP: 27400 + | > loss_disc: 2.7642006874084473 (2.67842482963455) + | > loss_disc_real_0: 0.11304240673780441 (0.2013519287596796) + | > loss_disc_real_1: 0.24455395340919495 (0.22362596165632534) + | > loss_disc_real_2: 0.2254514992237091 (0.2333740414442303) + | > loss_disc_real_3: 0.1886153221130371 (0.2340650876688066) + | > loss_disc_real_4: 0.23616521060466766 (0.23052662645823488) + | > loss_disc_real_5: 0.23096796870231628 (0.2388283399360202) + | > loss_0: 2.7642006874084473 (2.67842482963455) + | > grad_norm_0: tensor(27.0073, device='cuda:0') (tensor(10.5034, device='cuda:0')) + | > loss_gen: 2.074967384338379 (1.971015707911732) + | > loss_kl: 2.564815044403076 (2.546645200141124) + | > loss_feat: 4.129911422729492 (4.14235026591292) + | > loss_mel: 20.61919403076172 (20.631839850238542) + | > loss_duration: 1.3444206714630127 (1.3383897108452336) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.73331069946289 (30.63024076123104) + | > grad_norm_1: tensor(60.4987, device='cuda:0') (tensor(279.1745, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.8954 (0.8343373826731031) + | > loader_time: 0.0127 (0.011410183995683625) + + + --> TIME: 2024-06-25 10:20:58 -- STEP: 239/591 -- GLOBAL_STEP: 27425 + | > loss_disc: 2.6565511226654053 (2.6764193369254903) + | > loss_disc_real_0: 0.26361238956451416 (0.20119444701711503) + | > loss_disc_real_1: 0.1774384081363678 (0.2234713806766845) + | > loss_disc_real_2: 0.2131763994693756 (0.23325079099653157) + | > loss_disc_real_3: 0.2418331652879715 (0.23396512473228087) + | > loss_disc_real_4: 0.2362777441740036 (0.2305483033716928) + | > loss_disc_real_5: 0.27860984206199646 (0.2389304583666215) + | > loss_0: 2.6565511226654053 (2.6764193369254903) + | > grad_norm_0: tensor(13.3928, device='cuda:0') (tensor(10.3743, device='cuda:0')) + | > loss_gen: 1.9087063074111938 (1.970468576483148) + | > loss_kl: 2.4425485134124756 (2.540688223419829) + | > loss_feat: 3.9843664169311523 (4.134985707294992) + | > loss_mel: 20.5811767578125 (20.611507152413715) + | > loss_duration: 1.3303074836730957 (1.3383290972170965) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.24710464477539 (30.595978772789863) + | > grad_norm_1: tensor(322.6163, device='cuda:0') (tensor(283.2643, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.9074 (0.8391450010084208) + | > loader_time: 0.015 (0.011692325440410796) + + + --> TIME: 2024-06-25 10:21:21 -- STEP: 264/591 -- GLOBAL_STEP: 27450 + | > loss_disc: 2.66610050201416 (2.6790826194214095) + | > loss_disc_real_0: 0.2260880470275879 (0.20118382669082194) + | > loss_disc_real_1: 0.25126275420188904 (0.22402821108698845) + | > loss_disc_real_2: 0.2359054535627365 (0.23361778247988585) + | > loss_disc_real_3: 0.26154521107673645 (0.2341571691812891) + | > loss_disc_real_4: 0.23645621538162231 (0.23087719466650128) + | > loss_disc_real_5: 0.21618887782096863 (0.23892488133049372) + | > loss_0: 2.66610050201416 (2.6790826194214095) + | > grad_norm_0: tensor(8.5236, device='cuda:0') (tensor(10.3865, device='cuda:0')) + | > loss_gen: 2.0057010650634766 (1.9689379907918698) + | > loss_kl: 2.4469029903411865 (2.5386386170531776) + | > loss_feat: 3.93769907951355 (4.125947074456649) + | > loss_mel: 20.40679168701172 (20.599279172492743) + | > loss_duration: 1.3727641105651855 (1.338408958279725) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.169858932495117 (30.571211822105177) + | > grad_norm_1: tensor(409.6942, device='cuda:0') (tensor(278.0483, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.8992 (0.8445532349022952) + | > loader_time: 0.0131 (0.01197203903487235) + + + --> TIME: 2024-06-25 10:21:44 -- STEP: 289/591 -- GLOBAL_STEP: 27475 + | > loss_disc: 2.6768600940704346 (2.681357204707847) + | > loss_disc_real_0: 0.17726276814937592 (0.20148913763386156) + | > loss_disc_real_1: 0.20893557369709015 (0.22410580814915956) + | > loss_disc_real_2: 0.26561474800109863 (0.23390791516196768) + | > loss_disc_real_3: 0.2385188192129135 (0.23423342561433058) + | > loss_disc_real_4: 0.2248052954673767 (0.2311268130697594) + | > loss_disc_real_5: 0.24875973165035248 (0.23897662815544432) + | > loss_0: 2.6768600940704346 (2.681357204707847) + | > grad_norm_0: tensor(5.6295, device='cuda:0') (tensor(10.3541, device='cuda:0')) + | > loss_gen: 1.9958781003952026 (1.9669472936115462) + | > loss_kl: 2.499856948852539 (2.5355738537534305) + | > loss_feat: 3.773311138153076 (4.1128750705389) + | > loss_mel: 19.604951858520508 (20.60595094621387) + | > loss_duration: 1.3589752912521362 (1.3393487167193392) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.232973098754883 (30.56069587918714) + | > grad_norm_1: tensor(70.4248, device='cuda:0') (tensor(267.9121, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.9002 (0.8498416384198675) + | > loader_time: 0.0142 (0.012255864984848926) + + + --> TIME: 2024-06-25 10:22:08 -- STEP: 314/591 -- GLOBAL_STEP: 27500 + | > loss_disc: 2.6855392456054688 (2.680587159600226) + | > loss_disc_real_0: 0.17590677738189697 (0.20137805031363368) + | > loss_disc_real_1: 0.25440120697021484 (0.22410346989988522) + | > loss_disc_real_2: 0.2518289387226105 (0.23366877683408702) + | > loss_disc_real_3: 0.26611092686653137 (0.2340675957833126) + | > loss_disc_real_4: 0.23417706787586212 (0.23140953419504656) + | > loss_disc_real_5: 0.23221518099308014 (0.23894989191536692) + | > loss_0: 2.6855392456054688 (2.680587159600226) + | > grad_norm_0: tensor(9.6540, device='cuda:0') (tensor(10.2522, device='cuda:0')) + | > loss_gen: 1.8928725719451904 (1.966960650720414) + | > loss_kl: 2.5292563438415527 (2.532940878230296) + | > loss_feat: 3.7938220500946045 (4.112268768298398) + | > loss_mel: 20.79987335205078 (20.6153375601313) + | > loss_duration: 1.3652321100234985 (1.3397346621106367) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.38105583190918 (30.567242494813957) + | > grad_norm_1: tensor(309.0107, device='cuda:0') (tensor(257.0780, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.9174 (0.8559226830294178) + | > loader_time: 0.0145 (0.012533196218454161) + + + --> TIME: 2024-06-25 10:22:32 -- STEP: 339/591 -- GLOBAL_STEP: 27525 + | > loss_disc: 2.7181732654571533 (2.6808941940994018) + | > loss_disc_real_0: 0.15477705001831055 (0.20135108168322072) + | > loss_disc_real_1: 0.20585624873638153 (0.22403028299674876) + | > loss_disc_real_2: 0.22868213057518005 (0.23377364104652124) + | > loss_disc_real_3: 0.25882115960121155 (0.2340577084757937) + | > loss_disc_real_4: 0.23217962682247162 (0.23129030701685097) + | > loss_disc_real_5: 0.2462434470653534 (0.2389327565037747) + | > loss_0: 2.7181732654571533 (2.6808941940994018) + | > grad_norm_0: tensor(15.2728, device='cuda:0') (tensor(10.2751, device='cuda:0')) + | > loss_gen: 2.100768566131592 (1.9673374064904048) + | > loss_kl: 2.49967622756958 (2.5316020085045032) + | > loss_feat: 4.057365417480469 (4.108308176727069) + | > loss_mel: 20.41728973388672 (20.617234396020226) + | > loss_duration: 1.3771547079086304 (1.3412418267016555) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.452255249023438 (30.56572379615806) + | > grad_norm_1: tensor(64.5983, device='cuda:0') (tensor(257.1191, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.9125 (0.8610839428803211) + | > loader_time: 0.0147 (0.012790397205183997) + + + --> TIME: 2024-06-25 10:22:56 -- STEP: 364/591 -- GLOBAL_STEP: 27550 + | > loss_disc: 2.6304564476013184 (2.6817041288365364) + | > loss_disc_real_0: 0.23660504817962646 (0.2016910765279125) + | > loss_disc_real_1: 0.22442354261875153 (0.22425787531576313) + | > loss_disc_real_2: 0.22058013081550598 (0.23389333785398975) + | > loss_disc_real_3: 0.2247651070356369 (0.234043332778818) + | > loss_disc_real_4: 0.22334352135658264 (0.23123434312887248) + | > loss_disc_real_5: 0.22978229820728302 (0.2389849793616232) + | > loss_0: 2.6304564476013184 (2.6817041288365364) + | > grad_norm_0: tensor(10.1058, device='cuda:0') (tensor(10.2630, device='cuda:0')) + | > loss_gen: 1.8603819608688354 (1.9665419515017624) + | > loss_kl: 2.5794665813446045 (2.5317538688471037) + | > loss_feat: 4.0314621925354 (4.101242835050101) + | > loss_mel: 20.614160537719727 (20.616185355972455) + | > loss_duration: 1.3978211879730225 (1.3426237427271317) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.483293533325195 (30.55834773870615) + | > grad_norm_1: tensor(345.8425, device='cuda:0') (tensor(254.2555, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.9457 (0.866602918603918) + | > loader_time: 0.0179 (0.013090894117460151) + + + --> TIME: 2024-06-25 10:23:20 -- STEP: 389/591 -- GLOBAL_STEP: 27575 + | > loss_disc: 2.7287206649780273 (2.6829560489458397) + | > loss_disc_real_0: 0.18736079335212708 (0.20179716124172992) + | > loss_disc_real_1: 0.2583802044391632 (0.22452016370155511) + | > loss_disc_real_2: 0.2722052037715912 (0.23392206304024302) + | > loss_disc_real_3: 0.26994702219963074 (0.23422246581484543) + | > loss_disc_real_4: 0.21725523471832275 (0.2313345882917125) + | > loss_disc_real_5: 0.23582741618156433 (0.23895504239439044) + | > loss_0: 2.7287206649780273 (2.6829560489458397) + | > grad_norm_0: tensor(10.1334, device='cuda:0') (tensor(10.3438, device='cuda:0')) + | > loss_gen: 2.0796303749084473 (1.9662534234456355) + | > loss_kl: 2.600490093231201 (2.5292040842051384) + | > loss_feat: 3.891056537628174 (4.091718163159326) + | > loss_mel: 21.104475021362305 (20.600454202960876) + | > loss_duration: 1.3338488340377808 (1.3434173055663499) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 31.00950050354004 (30.53104716891801) + | > grad_norm_1: tensor(506.3800, device='cuda:0') (tensor(255.4617, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.9809 (0.8726744394375916) + | > loader_time: 0.0192 (0.013346372962304441) + + + --> TIME: 2024-06-25 10:23:45 -- STEP: 414/591 -- GLOBAL_STEP: 27600 + | > loss_disc: 2.6836094856262207 (2.682858873104702) + | > loss_disc_real_0: 0.24912407994270325 (0.20193484222183478) + | > loss_disc_real_1: 0.26698553562164307 (0.22452620402482396) + | > loss_disc_real_2: 0.2608267664909363 (0.23407298081738936) + | > loss_disc_real_3: 0.22458769381046295 (0.23402276570382324) + | > loss_disc_real_4: 0.25082722306251526 (0.23130931305712552) + | > loss_disc_real_5: 0.2644801437854767 (0.2388925630113353) + | > loss_0: 2.6836094856262207 (2.682858873104702) + | > grad_norm_0: tensor(10.9737, device='cuda:0') (tensor(10.4875, device='cuda:0')) + | > loss_gen: 1.8447633981704712 (1.9650288241496985) + | > loss_kl: 2.525374412536621 (2.527265867749275) + | > loss_feat: 3.8857109546661377 (4.080811795981033) + | > loss_mel: 20.580066680908203 (20.590202870576274) + | > loss_duration: 1.3894858360290527 (1.3445492977105464) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.225400924682617 (30.507858635722727) + | > grad_norm_1: tensor(267.0053, device='cuda:0') (tensor(257.6544, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.9683 (0.8790438872028664) + | > loader_time: 0.0178 (0.013614640143758436) + + + --> TIME: 2024-06-25 10:24:11 -- STEP: 439/591 -- GLOBAL_STEP: 27625 + | > loss_disc: 2.6742477416992188 (2.683489464953167) + | > loss_disc_real_0: 0.1178007572889328 (0.20162993908810992) + | > loss_disc_real_1: 0.22305718064308167 (0.224671020866255) + | > loss_disc_real_2: 0.22045892477035522 (0.23412502623228928) + | > loss_disc_real_3: 0.19270911812782288 (0.23407557864667072) + | > loss_disc_real_4: 0.20693251490592957 (0.231377075602362) + | > loss_disc_real_5: 0.22153717279434204 (0.2389069331941409) + | > loss_0: 2.6742477416992188 (2.683489464953167) + | > grad_norm_0: tensor(17.6933, device='cuda:0') (tensor(10.4806, device='cuda:0')) + | > loss_gen: 1.936951756477356 (1.9644042481594042) + | > loss_kl: 2.466360569000244 (2.526619950296668) + | > loss_feat: 3.7375710010528564 (4.073352617120415) + | > loss_mel: 19.626127243041992 (20.585525430144912) + | > loss_duration: 1.3639073371887207 (1.3454182859433799) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.130918502807617 (30.4953205351949) + | > grad_norm_1: tensor(60.0631, device='cuda:0') (tensor(255.7977, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 0.9841 (0.8859815966838585) + | > loader_time: 0.0187 (0.01388283199492783) + + + --> TIME: 2024-06-25 10:24:37 -- STEP: 464/591 -- GLOBAL_STEP: 27650 + | > loss_disc: 2.7118945121765137 (2.6858015625641247) + | > loss_disc_real_0: 0.14586573839187622 (0.20184595987264964) + | > loss_disc_real_1: 0.21739624440670013 (0.2247452105170694) + | > loss_disc_real_2: 0.22415240108966827 (0.23433802536977777) + | > loss_disc_real_3: 0.24661746621131897 (0.2344707901236312) + | > loss_disc_real_4: 0.23131193220615387 (0.23163836272754548) + | > loss_disc_real_5: 0.24406251311302185 (0.23907812549893198) + | > loss_0: 2.7118945121765137 (2.6858015625641247) + | > grad_norm_0: tensor(8.9205, device='cuda:0') (tensor(10.4936, device='cuda:0')) + | > loss_gen: 1.8838539123535156 (1.9618099727507294) + | > loss_kl: 2.5670011043548584 (2.5239171986949867) + | > loss_feat: 3.6887264251708984 (4.057508942382087) + | > loss_mel: 19.952430725097656 (20.565936080340677) + | > loss_duration: 1.3992266654968262 (1.3465929640264334) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.491239547729492 (30.45576515691034) + | > grad_norm_1: tensor(85.4984, device='cuda:0') (tensor(248.3875, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 1.0157 (0.8937035814441484) + | > loader_time: 0.0241 (0.014136635537805232) + + + --> TIME: 2024-06-25 10:25:04 -- STEP: 489/591 -- GLOBAL_STEP: 27675 + | > loss_disc: 2.6896402835845947 (2.6861017252526143) + | > loss_disc_real_0: 0.15002234280109406 (0.2016174245496225) + | > loss_disc_real_1: 0.1936095654964447 (0.22470535691408536) + | > loss_disc_real_2: 0.24801579117774963 (0.23436791448261596) + | > loss_disc_real_3: 0.23195958137512207 (0.23457896002474982) + | > loss_disc_real_4: 0.25064411759376526 (0.23176412802410518) + | > loss_disc_real_5: 0.242536261677742 (0.23907591832196054) + | > loss_0: 2.6896402835845947 (2.6861017252526143) + | > grad_norm_0: tensor(12.7791, device='cuda:0') (tensor(10.4185, device='cuda:0')) + | > loss_gen: 2.0565149784088135 (1.959768359890257) + | > loss_kl: 2.5288891792297363 (2.520619103757395) + | > loss_feat: 3.8942482471466064 (4.045962060156042) + | > loss_mel: 20.27485466003418 (20.5540302136193) + | > loss_duration: 1.3946011066436768 (1.347951645987652) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.14910888671875 (30.428331382923087) + | > grad_norm_1: tensor(401.1400, device='cuda:0') (tensor(244.7599, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 1.0463 (0.9012530654485972) + | > loader_time: 0.0168 (0.014392572434158172) + + + --> TIME: 2024-06-25 10:25:31 -- STEP: 514/591 -- GLOBAL_STEP: 27700 + | > loss_disc: 2.679741621017456 (2.6862977177252554) + | > loss_disc_real_0: 0.16576048731803894 (0.20171880969915407) + | > loss_disc_real_1: 0.23884901404380798 (0.22489770023507366) + | > loss_disc_real_2: 0.2414063960313797 (0.23437903079309352) + | > loss_disc_real_3: 0.23665115237236023 (0.23452557408856048) + | > loss_disc_real_4: 0.2155817598104477 (0.23158085346221927) + | > loss_disc_real_5: 0.2252945601940155 (0.23899587887386403) + | > loss_0: 2.679741621017456 (2.6862977177252554) + | > grad_norm_0: tensor(5.0023, device='cuda:0') (tensor(10.4584, device='cuda:0')) + | > loss_gen: 1.8020334243774414 (1.9589341892806473) + | > loss_kl: 2.5002079010009766 (2.517308989387543) + | > loss_feat: 3.9083211421966553 (4.038113089387053) + | > loss_mel: 20.120254516601562 (20.54018975788517) + | > loss_duration: 1.39588463306427 (1.3497162628266588) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.726703643798828 (30.404262286679753) + | > grad_norm_1: tensor(280.8490, device='cuda:0') (tensor(245.8692, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 1.1085 (0.9088242856445016) + | > loader_time: 0.0191 (0.014648980203769554) + + + --> TIME: 2024-06-25 10:25:59 -- STEP: 539/591 -- GLOBAL_STEP: 27725 + | > loss_disc: 2.6579999923706055 (2.6871377886558054) + | > loss_disc_real_0: 0.20514047145843506 (0.20183771413255042) + | > loss_disc_real_1: 0.2432873249053955 (0.2248938521836817) + | > loss_disc_real_2: 0.22423933446407318 (0.23452656982891634) + | > loss_disc_real_3: 0.32374921441078186 (0.23458824642624615) + | > loss_disc_real_4: 0.23337620496749878 (0.2316257872917657) + | > loss_disc_real_5: 0.2175440937280655 (0.23899419669317623) + | > loss_0: 2.6579999923706055 (2.6871377886558054) + | > grad_norm_0: tensor(7.7709, device='cuda:0') (tensor(10.5738, device='cuda:0')) + | > loss_gen: 2.0243749618530273 (1.958104176733622) + | > loss_kl: 2.4108428955078125 (2.5127962608726664) + | > loss_feat: 3.6679446697235107 (4.028344676313239) + | > loss_mel: 19.701854705810547 (20.52415882457386) + | > loss_duration: 1.3838434219360352 (1.351308046776199) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.188861846923828 (30.374711983279084) + | > grad_norm_1: tensor(123.9719, device='cuda:0') (tensor(241.8232, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 1.1601 (0.9182797314284684) + | > loader_time: 0.0186 (0.014945183260321399) + + + --> TIME: 2024-06-25 10:26:30 -- STEP: 564/591 -- GLOBAL_STEP: 27750 + | > loss_disc: 2.7223010063171387 (2.6882397883327274) + | > loss_disc_real_0: 0.14265340566635132 (0.20179282658541872) + | > loss_disc_real_1: 0.24406525492668152 (0.2248877108572645) + | > loss_disc_real_2: 0.2428026795387268 (0.2346990443501912) + | > loss_disc_real_3: 0.23898890614509583 (0.23467539308975774) + | > loss_disc_real_4: 0.2206994593143463 (0.23181038754100494) + | > loss_disc_real_5: 0.22848014533519745 (0.23909889441644047) + | > loss_0: 2.7223010063171387 (2.6882397883327274) + | > grad_norm_0: tensor(12.3847, device='cuda:0') (tensor(10.5235, device='cuda:0')) + | > loss_gen: 1.976499080657959 (1.9563345437776958) + | > loss_kl: 2.4614508152008057 (2.511864593265751) + | > loss_feat: 3.963330030441284 (4.018434145771864) + | > loss_mel: 20.187971115112305 (20.513005456180434) + | > loss_duration: 1.41458261013031 (1.3529834244268153) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 30.003833770751953 (30.352622153911184) + | > grad_norm_1: tensor(95.9244, device='cuda:0') (tensor(238.6611, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 1.1528 (0.9304118870843386) + | > loader_time: 0.0298 (0.015270659264097826) + + + --> TIME: 2024-06-25 10:27:02 -- STEP: 589/591 -- GLOBAL_STEP: 27775 + | > loss_disc: 2.7527620792388916 (2.688733557489001) + | > loss_disc_real_0: 0.1871757060289383 (0.20186524117185212) + | > loss_disc_real_1: 0.2406424880027771 (0.22506585363014034) + | > loss_disc_real_2: 0.22915203869342804 (0.234705900953662) + | > loss_disc_real_3: 0.18387076258659363 (0.23459093826735972) + | > loss_disc_real_4: 0.23977746069431305 (0.23187000695331392) + | > loss_disc_real_5: 0.25043541193008423 (0.23909500651124782) + | > loss_0: 2.7527620792388916 (2.688733557489001) + | > grad_norm_0: tensor(8.0602, device='cuda:0') (tensor(10.6169, device='cuda:0')) + | > loss_gen: 1.8626456260681152 (1.9556325020328644) + | > loss_kl: 2.5246613025665283 (2.511012143311558) + | > loss_feat: 3.6995058059692383 (4.008820709834073) + | > loss_mel: 19.76406478881836 (20.50086535986493) + | > loss_duration: 1.3925340175628662 (1.3547382674516757) + | > amp_scaler: 256.0 (256.0) + | > loss_1: 29.243412017822266 (30.33106897156591) + | > grad_norm_1: tensor(159.7301, device='cuda:0') (tensor(239.8280, device='cuda:0')) + | > current_lr_0: 0.00019885322845327182 + | > current_lr_1: 0.00019885322845327182 + | > step_time: 1.2432 (0.9435380472594489) + | > loader_time: 0.0211 (0.015596087395841281) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.00357804907129166 (-0.0002021891005495758) + | > avg_loss_disc: 2.7968469782078524 (+0.1312332356229744) + | > avg_loss_disc_real_0: 0.3133753557154473 (+0.016778762353227505) + | > avg_loss_disc_real_1: 0.2254637780975788 (+0.004912208052391703) + | > avg_loss_disc_real_2: 0.23737647368552836 (+0.012275268739842332) + | > avg_loss_disc_real_3: 0.2273697868940678 (+0.029326224263678202) + | > avg_loss_disc_real_4: 0.25537991809084043 (+0.012693147392982967) + | > avg_loss_disc_real_5: 0.2345985694134489 (+0.004468499663028314) + | > avg_loss_0: 2.7968469782078524 (+0.1312332356229744) + | > avg_loss_gen: 1.9838429537225277 (-0.058598031388952254) + | > avg_loss_kl: 2.5647682129068565 (-0.004851452847745907) + | > avg_loss_feat: 3.8792870349072395 (-0.10345795814027348) + | > avg_loss_mel: 20.674018900445166 (-0.13116921769811896) + | > avg_loss_duration: 1.3676708201144605 (-0.0028000943204189443) + | > avg_loss_1: 30.46958805652375 (-0.3008766174316406) + + + > EPOCH: 48/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 10:27:12)  + + --> TIME: 2024-06-25 10:27:32 -- STEP: 23/591 -- GLOBAL_STEP: 27800 + | > loss_disc: 2.6351914405822754 (2.6945112788158916) + | > loss_disc_real_0: 0.20554524660110474 (0.20096498922161435) + | > loss_disc_real_1: 0.2171136736869812 (0.22715105764243915) + | > loss_disc_real_2: 0.2122575044631958 (0.23336525909278705) + | > loss_disc_real_3: 0.20498071610927582 (0.23414201127446216) + | > loss_disc_real_4: 0.23767781257629395 (0.23482270538806915) + | > loss_disc_real_5: 0.24025343358516693 (0.2399093087600625) + | > loss_0: 2.6351914405822754 (2.6945112788158916) + | > grad_norm_0: tensor(4.0511, device='cuda:0') (tensor(7.1243, device='cuda:0')) + | > loss_gen: 1.9888864755630493 (1.961753275083459) + | > loss_kl: 2.668739080429077 (2.5965171896893042) + | > loss_feat: 4.2703938484191895 (4.226781824360723) + | > loss_mel: 20.751176834106445 (20.691124128258746) + | > loss_duration: 1.3151333332061768 (1.318352953247402) + | > amp_scaler: 512.0 (445.2173913043478) + | > loss_1: 30.99432945251465 (30.794529458750848) + | > grad_norm_1: tensor(139.9138, device='cuda:0') (tensor(158.9660, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.8054 (0.7771524761034094) + | > loader_time: 0.0091 (0.008907204088957411) + + + --> TIME: 2024-06-25 10:27:52 -- STEP: 48/591 -- GLOBAL_STEP: 27825 + | > loss_disc: 2.688506841659546 (2.685887043674787) + | > loss_disc_real_0: 0.18218931555747986 (0.19935785233974457) + | > loss_disc_real_1: 0.22205817699432373 (0.22558683436363935) + | > loss_disc_real_2: 0.23490175604820251 (0.23222593093911806) + | > loss_disc_real_3: 0.2338402271270752 (0.23437922665228447) + | > loss_disc_real_4: 0.23254406452178955 (0.23387014182905355) + | > loss_disc_real_5: 0.24654456973075867 (0.24028947545836368) + | > loss_0: 2.688506841659546 (2.685887043674787) + | > grad_norm_0: tensor(9.6054, device='cuda:0') (tensor(8.1528, device='cuda:0')) + | > loss_gen: 2.0751426219940186 (1.9667647058765094) + | > loss_kl: 2.4308788776397705 (2.5912937174240755) + | > loss_feat: 3.9309499263763428 (4.174074709415435) + | > loss_mel: 20.084468841552734 (20.638826092084255) + | > loss_duration: 1.3702030181884766 (1.328207656741142) + | > amp_scaler: 512.0 (480.0) + | > loss_1: 29.89164161682129 (30.699166774749756) + | > grad_norm_1: tensor(125.0238, device='cuda:0') (tensor(184.3313, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.8081 (0.7938044021526972) + | > loader_time: 0.0104 (0.009382123748461401) + + + --> TIME: 2024-06-25 10:28:13 -- STEP: 73/591 -- GLOBAL_STEP: 27850 + | > loss_disc: 2.682781457901001 (2.6819036758109314) + | > loss_disc_real_0: 0.2458883672952652 (0.19956079094785534) + | > loss_disc_real_1: 0.2173040360212326 (0.22498871447288826) + | > loss_disc_real_2: 0.2144538015127182 (0.23288615354119915) + | > loss_disc_real_3: 0.2368048131465912 (0.2345621298028998) + | > loss_disc_real_4: 0.2518470585346222 (0.23316786305545129) + | > loss_disc_real_5: 0.24341486394405365 (0.23952957559121799) + | > loss_0: 2.682781457901001 (2.6819036758109314) + | > grad_norm_0: tensor(7.4436, device='cuda:0') (tensor(8.3371, device='cuda:0')) + | > loss_gen: 1.9993464946746826 (1.961284141017966) + | > loss_kl: 2.4802916049957275 (2.560766856964321) + | > loss_feat: 4.020977020263672 (4.149475274020677) + | > loss_mel: 20.83526611328125 (20.631050580168427) + | > loss_duration: 1.3487043380737305 (1.3324925458594545) + | > amp_scaler: 512.0 (490.958904109589) + | > loss_1: 30.684585571289062 (30.63506944212195) + | > grad_norm_1: tensor(139.6918, device='cuda:0') (tensor(184.7969, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.8193 (0.8013488985087773) + | > loader_time: 0.0102 (0.00980163600346813) + + + --> TIME: 2024-06-25 10:28:34 -- STEP: 98/591 -- GLOBAL_STEP: 27875 + | > loss_disc: 2.6179046630859375 (2.6823819948702448) + | > loss_disc_real_0: 0.19452770054340363 (0.20064034776724116) + | > loss_disc_real_1: 0.23171846568584442 (0.22505967729553886) + | > loss_disc_real_2: 0.2104683369398117 (0.2333813972619115) + | > loss_disc_real_3: 0.2369501143693924 (0.23428049835623527) + | > loss_disc_real_4: 0.21098412573337555 (0.23224788812958463) + | > loss_disc_real_5: 0.22804346680641174 (0.23932241161867063) + | > loss_0: 2.6179046630859375 (2.6823819948702448) + | > grad_norm_0: tensor(5.0118, device='cuda:0') (tensor(8.7032, device='cuda:0')) + | > loss_gen: 2.0574817657470703 (1.965039955109966) + | > loss_kl: 2.5570311546325684 (2.5550698358185433) + | > loss_feat: 4.257920265197754 (4.162654728305581) + | > loss_mel: 21.80014991760254 (20.66362838355862) + | > loss_duration: 1.359491229057312 (1.3398640593703919) + | > amp_scaler: 512.0 (496.3265306122449) + | > loss_1: 32.032073974609375 (30.68625699257364) + | > grad_norm_1: tensor(251.3288, device='cuda:0') (tensor(189.3051, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.8311 (0.8088719187950602) + | > loader_time: 0.0107 (0.010132838268669288) + + + --> TIME: 2024-06-25 10:28:56 -- STEP: 123/591 -- GLOBAL_STEP: 27900 + | > loss_disc: 2.7848405838012695 (2.683813692108403) + | > loss_disc_real_0: 0.2858212888240814 (0.20189013235210404) + | > loss_disc_real_1: 0.2571212351322174 (0.22515873235415637) + | > loss_disc_real_2: 0.2613850235939026 (0.2334429013535259) + | > loss_disc_real_3: 0.2742725610733032 (0.2337962418067746) + | > loss_disc_real_4: 0.23772725462913513 (0.23116578706880894) + | > loss_disc_real_5: 0.2546115815639496 (0.23931319951041927) + | > loss_0: 2.7848405838012695 (2.683813692108403) + | > grad_norm_0: tensor(12.4534, device='cuda:0') (tensor(9.9014, device='cuda:0')) + | > loss_gen: 2.1271984577178955 (1.9683103987841102) + | > loss_kl: 2.545872688293457 (2.545099882575554) + | > loss_feat: 4.16414213180542 (4.162174548560041) + | > loss_mel: 20.7735652923584 (20.660646965833216) + | > loss_duration: 1.3502798080444336 (1.340819063225413) + | > amp_scaler: 256.0 (468.2926829268293) + | > loss_1: 30.9610595703125 (30.677050877392777) + | > grad_norm_1: tensor(448.8555, device='cuda:0') (tensor(215.1852, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.8483 (0.8146469980720582) + | > loader_time: 0.0132 (0.010435891345264469) + + + --> TIME: 2024-06-25 10:29:17 -- STEP: 148/591 -- GLOBAL_STEP: 27925 + | > loss_disc: 2.6829354763031006 (2.677263878487253) + | > loss_disc_real_0: 0.19591182470321655 (0.20082948613609816) + | > loss_disc_real_1: 0.24752891063690186 (0.2243335168506648) + | > loss_disc_real_2: 0.2401144802570343 (0.2334976193268557) + | > loss_disc_real_3: 0.24295803904533386 (0.23418323826548215) + | > loss_disc_real_4: 0.17535297572612762 (0.22884735414707982) + | > loss_disc_real_5: 0.22097447514533997 (0.23895441146718488) + | > loss_0: 2.6829354763031006 (2.677263878487253) + | > grad_norm_0: tensor(11.1857, device='cuda:0') (tensor(9.9749, device='cuda:0')) + | > loss_gen: 2.0692739486694336 (1.9766319482713133) + | > loss_kl: 2.4406774044036865 (2.5451136038110063) + | > loss_feat: 4.126019477844238 (4.175486946428144) + | > loss_mel: 20.668590545654297 (20.661146344365296) + | > loss_duration: 1.3608559370040894 (1.3404729914020852) + | > amp_scaler: 256.0 (432.4324324324325) + | > loss_1: 30.665416717529297 (30.698851804475527) + | > grad_norm_1: tensor(568.9767, device='cuda:0') (tensor(278.5381, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.8425 (0.8202804681417104) + | > loader_time: 0.0114 (0.010717775370623617) + + + --> TIME: 2024-06-25 10:29:39 -- STEP: 173/591 -- GLOBAL_STEP: 27950 + | > loss_disc: 2.6369152069091797 (2.6790559112681143) + | > loss_disc_real_0: 0.1494501382112503 (0.20121882812825242) + | > loss_disc_real_1: 0.20407621562480927 (0.22522261764617324) + | > loss_disc_real_2: 0.21366453170776367 (0.23407896242045254) + | > loss_disc_real_3: 0.23017926514148712 (0.23423405939099418) + | > loss_disc_real_4: 0.20020462572574615 (0.2282864511702102) + | > loss_disc_real_5: 0.21393412351608276 (0.23901289822049224) + | > loss_0: 2.6369152069091797 (2.6790559112681143) + | > grad_norm_0: tensor(10.0772, device='cuda:0') (tensor(10.2224, device='cuda:0')) + | > loss_gen: 1.9283846616744995 (1.9800470729783781) + | > loss_kl: 2.382598876953125 (2.5351829225617335) + | > loss_feat: 4.301138877868652 (4.177048407538089) + | > loss_mel: 21.073802947998047 (20.672427304218267) + | > loss_duration: 1.3094393014907837 (1.3386090366826584) + | > amp_scaler: 256.0 (406.93641618497116) + | > loss_1: 30.995365142822266 (30.703314720550715) + | > grad_norm_1: tensor(526.9727, device='cuda:0') (tensor(307.0030, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.87 (0.8261486519278819) + | > loader_time: 0.0116 (0.010911977360014287) + + + --> TIME: 2024-06-25 10:30:01 -- STEP: 198/591 -- GLOBAL_STEP: 27975 + | > loss_disc: 2.640523910522461 (2.6756512950165114) + | > loss_disc_real_0: 0.1810792088508606 (0.20069710383511552) + | > loss_disc_real_1: 0.2130597084760666 (0.22454006947351224) + | > loss_disc_real_2: 0.2347911298274994 (0.23356346913961448) + | > loss_disc_real_3: 0.2513924241065979 (0.23403075608340176) + | > loss_disc_real_4: 0.2196984887123108 (0.22910857155467523) + | > loss_disc_real_5: 0.22015714645385742 (0.23901683807072013) + | > loss_0: 2.640523910522461 (2.6756512950165114) + | > grad_norm_0: tensor(10.3413, device='cuda:0') (tensor(10.1748, device='cuda:0')) + | > loss_gen: 1.972014307975769 (1.9803295815833892) + | > loss_kl: 2.525940418243408 (2.531109183725684) + | > loss_feat: 4.247249126434326 (4.176538711846479) + | > loss_mel: 20.956809997558594 (20.648604161811587) + | > loss_duration: 1.3553205728530884 (1.3383097209111614) + | > amp_scaler: 256.0 (387.87878787878793) + | > loss_1: 31.05733299255371 (30.674891336999757) + | > grad_norm_1: tensor(97.5825, device='cuda:0') (tensor(302.6530, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.8627 (0.8314797613355849) + | > loader_time: 0.0119 (0.011083822057704737) + + + --> TIME: 2024-06-25 10:30:24 -- STEP: 223/591 -- GLOBAL_STEP: 28000 + | > loss_disc: 2.7024970054626465 (2.674803361764404) + | > loss_disc_real_0: 0.20499053597450256 (0.20026102147562086) + | > loss_disc_real_1: 0.21395127475261688 (0.22436114016402461) + | > loss_disc_real_2: 0.22536002099514008 (0.23335959872590054) + | > loss_disc_real_3: 0.20270606875419617 (0.23370589930647692) + | > loss_disc_real_4: 0.2692044973373413 (0.22961980211360572) + | > loss_disc_real_5: 0.27359631657600403 (0.23903644913515168) + | > loss_0: 2.7024970054626465 (2.674803361764404) + | > grad_norm_0: tensor(7.6024, device='cuda:0') (tensor(10.0265, device='cuda:0')) + | > loss_gen: 2.005620002746582 (1.9794965346297875) + | > loss_kl: 2.515279769897461 (2.5234689755290067) + | > loss_feat: 3.9431352615356445 (4.166544901415907) + | > loss_mel: 20.60774040222168 (20.624842092060714) + | > loss_duration: 1.3143876791000366 (1.3375340090738823) + | > amp_scaler: 256.0 (373.0941704035875) + | > loss_1: 30.38616180419922 (30.631886486515334) + | > grad_norm_1: tensor(311.1655, device='cuda:0') (tensor(293.1559, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.8886 (0.8365072936755125) + | > loader_time: 0.0149 (0.011450784623355612) + + + --> TIME: 2024-06-25 10:30:46 -- STEP: 248/591 -- GLOBAL_STEP: 28025 + | > loss_disc: 2.630307197570801 (2.6762046064099967) + | > loss_disc_real_0: 0.22203420102596283 (0.20045468511600648) + | > loss_disc_real_1: 0.20318593084812164 (0.22440535947680473) + | > loss_disc_real_2: 0.2420777827501297 (0.23358389188445383) + | > loss_disc_real_3: 0.23150481283664703 (0.2338759269445173) + | > loss_disc_real_4: 0.2271205484867096 (0.23000635597254) + | > loss_disc_real_5: 0.22466720640659332 (0.23919200080056344) + | > loss_0: 2.630307197570801 (2.6762046064099967) + | > grad_norm_0: tensor(7.4635, device='cuda:0') (tensor(10.1299, device='cuda:0')) + | > loss_gen: 1.8608684539794922 (1.9773028007438105) + | > loss_kl: 2.4611470699310303 (2.5255940796867486) + | > loss_feat: 4.174180030822754 (4.159653773230893) + | > loss_mel: 20.487741470336914 (20.622271183998347) + | > loss_duration: 1.3552005290985107 (1.3381295814629528) + | > amp_scaler: 256.0 (361.29032258064524) + | > loss_1: 30.33913803100586 (30.622951415277296) + | > grad_norm_1: tensor(357.2489, device='cuda:0') (tensor(287.5561, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.915 (0.8417268808810942) + | > loader_time: 0.0133 (0.011754746398618147) + + + --> TIME: 2024-06-25 10:31:09 -- STEP: 273/591 -- GLOBAL_STEP: 28050 + | > loss_disc: 2.7988393306732178 (2.677848141708654) + | > loss_disc_real_0: 0.36420589685440063 (0.20093249636037008) + | > loss_disc_real_1: 0.24555377662181854 (0.22469271959620954) + | > loss_disc_real_2: 0.22307389974594116 (0.2339765002657642) + | > loss_disc_real_3: 0.26204758882522583 (0.23408267114843645) + | > loss_disc_real_4: 0.20758162438869476 (0.22988579943503212) + | > loss_disc_real_5: 0.23766395449638367 (0.23906014624969427) + | > loss_0: 2.7988393306732178 (2.677848141708654) + | > grad_norm_0: tensor(24.1400, device='cuda:0') (tensor(10.3823, device='cuda:0')) + | > loss_gen: 1.800377607345581 (1.9780993893906311) + | > loss_kl: 2.5649302005767822 (2.5257120525443937) + | > loss_feat: 3.9323537349700928 (4.154161965890683) + | > loss_mel: 20.23900604248047 (20.61208249885083) + | > loss_duration: 1.336235523223877 (1.3380646312629787) + | > amp_scaler: 256.0 (351.64835164835176) + | > loss_1: 29.872901916503906 (30.608120516542986) + | > grad_norm_1: tensor(421.3367, device='cuda:0') (tensor(298.6340, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.8886 (0.8465428911285959) + | > loader_time: 0.0151 (0.012026958849840551) + + + --> TIME: 2024-06-25 10:31:32 -- STEP: 298/591 -- GLOBAL_STEP: 28075 + | > loss_disc: 2.6787543296813965 (2.6776670289519653) + | > loss_disc_real_0: 0.20245158672332764 (0.2010128395319386) + | > loss_disc_real_1: 0.20080050826072693 (0.22452826827964525) + | > loss_disc_real_2: 0.1912309229373932 (0.23380940112491583) + | > loss_disc_real_3: 0.18779335916042328 (0.23384709761246744) + | > loss_disc_real_4: 0.22467109560966492 (0.22944174411912893) + | > loss_disc_real_5: 0.23777416348457336 (0.23894936341367312) + | > loss_0: 2.6787543296813965 (2.6776670289519653) + | > grad_norm_0: tensor(8.0938, device='cuda:0') (tensor(10.4786, device='cuda:0')) + | > loss_gen: 2.107259511947632 (1.9785656632993045) + | > loss_kl: 2.5550498962402344 (2.522592448548182) + | > loss_feat: 4.385190486907959 (4.151248625460887) + | > loss_mel: 20.436988830566406 (20.61044545141642) + | > loss_duration: 1.3200421333312988 (1.3385386819007417) + | > amp_scaler: 256.0 (343.6241610738256) + | > loss_1: 30.80453109741211 (30.60139081942155) + | > grad_norm_1: tensor(416.6691, device='cuda:0') (tensor(312.6900, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.9118 (0.8514366941964068) + | > loader_time: 0.0167 (0.012273854057260811) + + + --> TIME: 2024-06-25 10:31:56 -- STEP: 323/591 -- GLOBAL_STEP: 28100 + | > loss_disc: 2.671767473220825 (2.6772103907523146) + | > loss_disc_real_0: 0.27494674921035767 (0.2013210828413167) + | > loss_disc_real_1: 0.2100246399641037 (0.22434818477084392) + | > loss_disc_real_2: 0.22042162716388702 (0.2339055719021304) + | > loss_disc_real_3: 0.2394513338804245 (0.2338922533549761) + | > loss_disc_real_4: 0.26813164353370667 (0.2293902046447937) + | > loss_disc_real_5: 0.25638532638549805 (0.23890403881899713) + | > loss_0: 2.671767473220825 (2.6772103907523146) + | > grad_norm_0: tensor(12.9503, device='cuda:0') (tensor(10.4256, device='cuda:0')) + | > loss_gen: 1.9545660018920898 (1.9782267011725128) + | > loss_kl: 2.74574613571167 (2.523822073596918) + | > loss_feat: 4.212590217590332 (4.1519189936457765) + | > loss_mel: 20.62605094909668 (20.617429798232504) + | > loss_duration: 1.370409607887268 (1.3392137089749985) + | > amp_scaler: 256.0 (336.84210526315786) + | > loss_1: 30.909360885620117 (30.610611227643748) + | > grad_norm_1: tensor(322.3457, device='cuda:0') (tensor(313.6720, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.9082 (0.8564231905036668) + | > loader_time: 0.016 (0.012531456932563906) + + + --> TIME: 2024-06-25 10:32:20 -- STEP: 348/591 -- GLOBAL_STEP: 28125 + | > loss_disc: 2.676898956298828 (2.678116859375746) + | > loss_disc_real_0: 0.1314094364643097 (0.20102736266094387) + | > loss_disc_real_1: 0.2147710919380188 (0.22455603779218664) + | > loss_disc_real_2: 0.26102927327156067 (0.23399254341406384) + | > loss_disc_real_3: 0.2766880989074707 (0.2339757982341723) + | > loss_disc_real_4: 0.24314771592617035 (0.22970401123166084) + | > loss_disc_real_5: 0.25475096702575684 (0.23909064270984168) + | > loss_0: 2.676898956298828 (2.678116859375746) + | > grad_norm_0: tensor(17.1589, device='cuda:0') (tensor(10.3163, device='cuda:0')) + | > loss_gen: 2.2680821418762207 (1.976814597502522) + | > loss_kl: 2.4028334617614746 (2.52290324986666) + | > loss_feat: 4.167602062225342 (4.1450076089508245) + | > loss_mel: 20.89282989501953 (20.61349412764625) + | > loss_duration: 1.3693779706954956 (1.3407416514966684) + | > amp_scaler: 256.0 (331.0344827586206) + | > loss_1: 31.100723266601562 (30.598961188875396) + | > grad_norm_1: tensor(504.5741, device='cuda:0') (tensor(308.7825, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.9422 (0.8617614288439698) + | > loader_time: 0.0169 (0.012813565374790942) + + + --> TIME: 2024-06-25 10:32:44 -- STEP: 373/591 -- GLOBAL_STEP: 28150 + | > loss_disc: 2.626563549041748 (2.677904929935773) + | > loss_disc_real_0: 0.17117473483085632 (0.201334864879901) + | > loss_disc_real_1: 0.2585173547267914 (0.22451548797675175) + | > loss_disc_real_2: 0.24528765678405762 (0.23398199222043117) + | > loss_disc_real_3: 0.1899278312921524 (0.23392608981509636) + | > loss_disc_real_4: 0.2389427274465561 (0.22966948738846) + | > loss_disc_real_5: 0.23440775275230408 (0.23891142593951392) + | > loss_0: 2.626563549041748 (2.677904929935773) + | > grad_norm_0: tensor(14.9054, device='cuda:0') (tensor(10.2932, device='cuda:0')) + | > loss_gen: 2.0937676429748535 (1.9756327669358444) + | > loss_kl: 2.4157602787017822 (2.521377543661613) + | > loss_feat: 3.8949434757232666 (4.138963021797408) + | > loss_mel: 19.928043365478516 (20.601646367090943) + | > loss_duration: 1.3373236656188965 (1.3423258580726856) + | > amp_scaler: 256.0 (326.00536193029484) + | > loss_1: 29.669837951660156 (30.57994554125911) + | > grad_norm_1: tensor(519.1237, device='cuda:0') (tensor(313.6269, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.9699 (0.8672740389131031) + | > loader_time: 0.0181 (0.013100021965701847) + + + --> TIME: 2024-06-25 10:33:08 -- STEP: 398/591 -- GLOBAL_STEP: 28175 + | > loss_disc: 2.7073097229003906 (2.6767158592166624) + | > loss_disc_real_0: 0.25286388397216797 (0.20142712202578342) + | > loss_disc_real_1: 0.24146591126918793 (0.2244372016670716) + | > loss_disc_real_2: 0.23208151757717133 (0.23405819498564132) + | > loss_disc_real_3: 0.21312162280082703 (0.23405776775662027) + | > loss_disc_real_4: 0.22524486482143402 (0.228948656638064) + | > loss_disc_real_5: 0.24929751455783844 (0.23883909855655688) + | > loss_0: 2.7073097229003906 (2.6767158592166624) + | > grad_norm_0: tensor(11.9516, device='cuda:0') (tensor(10.5144, device='cuda:0')) + | > loss_gen: 1.8245372772216797 (1.978413364995065) + | > loss_kl: 2.376070976257324 (2.518292480377695) + | > loss_feat: 3.998579263687134 (4.137314949203377) + | > loss_mel: 20.478683471679688 (20.59688576741433) + | > loss_duration: 1.3426733016967773 (1.3431761869833103) + | > amp_scaler: 256.0 (321.6080402010049) + | > loss_1: 30.02054214477539 (30.57408273399775) + | > grad_norm_1: tensor(435.5615, device='cuda:0') (tensor(328.8247, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.9545 (0.8732176060652614) + | > loader_time: 0.0187 (0.013385416874334446) + + + --> TIME: 2024-06-25 10:33:33 -- STEP: 423/591 -- GLOBAL_STEP: 28200 + | > loss_disc: 2.6975066661834717 (2.676347614180113) + | > loss_disc_real_0: 0.23512513935565948 (0.20141297608754885) + | > loss_disc_real_1: 0.1882595270872116 (0.22430054152970055) + | > loss_disc_real_2: 0.24265477061271667 (0.23415275689557935) + | > loss_disc_real_3: 0.22085946798324585 (0.23392102330148082) + | > loss_disc_real_4: 0.21536433696746826 (0.22864675609918914) + | > loss_disc_real_5: 0.2705596089363098 (0.23899798761022853) + | > loss_0: 2.6975066661834717 (2.676347614180113) + | > grad_norm_0: tensor(12.1831, device='cuda:0') (tensor(10.7546, device='cuda:0')) + | > loss_gen: 2.0679168701171875 (1.9785745149526754) + | > loss_kl: 2.447783946990967 (2.515964348547283) + | > loss_feat: 3.9514031410217285 (4.127798549954212) + | > loss_mel: 20.73819923400879 (20.578053043807362) + | > loss_duration: 1.379746913909912 (1.3440996242189525) + | > amp_scaler: 256.0 (317.73049645390057) + | > loss_1: 30.585050582885742 (30.544490065698646) + | > grad_norm_1: tensor(381.0150, device='cuda:0') (tensor(339.3698, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 0.9666 (0.8793323321827198) + | > loader_time: 0.0192 (0.013651031128903658) + + + --> TIME: 2024-06-25 10:33:59 -- STEP: 448/591 -- GLOBAL_STEP: 28225 + | > loss_disc: 2.6976120471954346 (2.675946922706707) + | > loss_disc_real_0: 0.16796699166297913 (0.20133624759702284) + | > loss_disc_real_1: 0.22534127533435822 (0.22451973546828544) + | > loss_disc_real_2: 0.22305527329444885 (0.23413221105667098) + | > loss_disc_real_3: 0.23569022119045258 (0.2340369608386287) + | > loss_disc_real_4: 0.23256109654903412 (0.22840970691426524) + | > loss_disc_real_5: 0.24832206964492798 (0.23899784045560019) + | > loss_0: 2.6976120471954346 (2.675946922706707) + | > grad_norm_0: tensor(9.8094, device='cuda:0') (tensor(10.8404, device='cuda:0')) + | > loss_gen: 1.9478514194488525 (1.980716488723244) + | > loss_kl: 2.362837314605713 (2.514567950474363) + | > loss_feat: 3.8540596961975098 (4.126140171928068) + | > loss_mel: 20.255155563354492 (20.572629775319772) + | > loss_duration: 1.388587236404419 (1.3453334793448453) + | > amp_scaler: 128.0 (307.142857142857) + | > loss_1: 29.80849266052246 (30.53938782640866) + | > grad_norm_1: tensor(219.9049, device='cuda:0') (tensor(346.9725, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 1.0052 (0.8866451643407346) + | > loader_time: 0.0167 (0.013936121016740802) + + + --> TIME: 2024-06-25 10:34:25 -- STEP: 473/591 -- GLOBAL_STEP: 28250 + | > loss_disc: 2.6469650268554688 (2.67701756324123) + | > loss_disc_real_0: 0.2644445300102234 (0.2016294396011825) + | > loss_disc_real_1: 0.23250386118888855 (0.22463832742063994) + | > loss_disc_real_2: 0.22913533449172974 (0.23420836362223796) + | > loss_disc_real_3: 0.22182676196098328 (0.23407968151014907) + | > loss_disc_real_4: 0.22039341926574707 (0.22848626605804073) + | > loss_disc_real_5: 0.2091459482908249 (0.2388922780689175) + | > loss_0: 2.6469650268554688 (2.67701756324123) + | > grad_norm_0: tensor(15.1185, device='cuda:0') (tensor(10.9651, device='cuda:0')) + | > loss_gen: 1.8992416858673096 (1.9789248818322922) + | > loss_kl: 2.4342806339263916 (2.510980892887074) + | > loss_feat: 4.048898220062256 (4.113710904978299) + | > loss_mel: 21.268901824951172 (20.555174627969425) + | > loss_duration: 1.3817201852798462 (1.3465979663816623) + | > amp_scaler: 128.0 (297.67441860465107) + | > loss_1: 31.033042907714844 (30.505389235740484) + | > grad_norm_1: tensor(122.8060, device='cuda:0') (tensor(346.7318, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 1.0437 (0.8934881540988012) + | > loader_time: 0.023 (0.014234744469156733) + + + --> TIME: 2024-06-25 10:34:52 -- STEP: 498/591 -- GLOBAL_STEP: 28275 + | > loss_disc: 2.679649591445923 (2.6767596135656526) + | > loss_disc_real_0: 0.0844024121761322 (0.201257012455339) + | > loss_disc_real_1: 0.20653867721557617 (0.22456611022652392) + | > loss_disc_real_2: 0.2627626061439514 (0.23417752780708442) + | > loss_disc_real_3: 0.23171760141849518 (0.23408583363136615) + | > loss_disc_real_4: 0.23889504373073578 (0.2286320215308044) + | > loss_disc_real_5: 0.24163834750652313 (0.2389240218214242) + | > loss_0: 2.679649591445923 (2.6767596135656526) + | > grad_norm_0: tensor(30.8448, device='cuda:0') (tensor(10.9890, device='cuda:0')) + | > loss_gen: 2.106494188308716 (1.978031652758878) + | > loss_kl: 2.423269748687744 (2.5069520396879867) + | > loss_feat: 4.259997844696045 (4.107380570178053) + | > loss_mel: 20.577638626098633 (20.530767961678247) + | > loss_duration: 1.3726468086242676 (1.3479086397642117) + | > amp_scaler: 128.0 (289.156626506024) + | > loss_1: 30.740047454833984 (30.47104085209858) + | > grad_norm_1: tensor(89.9557, device='cuda:0') (tensor(340.9165, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 1.09 (0.9014892099372834) + | > loader_time: 0.0169 (0.014533108018009541) + + + --> TIME: 2024-06-25 10:35:20 -- STEP: 523/591 -- GLOBAL_STEP: 28300 + | > loss_disc: 2.7230753898620605 (2.677300553932701) + | > loss_disc_real_0: 0.16208875179290771 (0.2015075508939605) + | > loss_disc_real_1: 0.23102088272571564 (0.22454947258842606) + | > loss_disc_real_2: 0.21937119960784912 (0.23406476947705102) + | > loss_disc_real_3: 0.2258450984954834 (0.23406230642622336) + | > loss_disc_real_4: 0.22784243524074554 (0.2288485278818849) + | > loss_disc_real_5: 0.2349843978881836 (0.2389102891571891) + | > loss_0: 2.7230753898620605 (2.677300553932701) + | > grad_norm_0: tensor(13.3907, device='cuda:0') (tensor(10.9272, device='cuda:0')) + | > loss_gen: 1.9549241065979004 (1.9763098041140554) + | > loss_kl: 2.460606813430786 (2.5047689929291113) + | > loss_feat: 3.9722821712493896 (4.101166770289552) + | > loss_mel: 20.120405197143555 (20.52142541495153) + | > loss_duration: 1.3880428075790405 (1.349875954779339) + | > amp_scaler: 128.0 (281.4531548757169) + | > loss_1: 29.89626121520996 (30.45354691791717) + | > grad_norm_1: tensor(115.2492, device='cuda:0') (tensor(333.8229, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 1.1004 (0.9112073048578851) + | > loader_time: 0.0239 (0.014849482257325157) + + + --> TIME: 2024-06-25 10:35:50 -- STEP: 548/591 -- GLOBAL_STEP: 28325 + | > loss_disc: 2.6454787254333496 (2.6782818536688815) + | > loss_disc_real_0: 0.1667640507221222 (0.20165072028001732) + | > loss_disc_real_1: 0.23662418127059937 (0.22461052035001947) + | > loss_disc_real_2: 0.22979871928691864 (0.23411653585133763) + | > loss_disc_real_3: 0.2570817768573761 (0.23410319982871522) + | > loss_disc_real_4: 0.19120492041110992 (0.2290787061907514) + | > loss_disc_real_5: 0.21023470163345337 (0.2389133821928153) + | > loss_0: 2.6454787254333496 (2.6782818536688815) + | > grad_norm_0: tensor(12.8178, device='cuda:0') (tensor(10.9466, device='cuda:0')) + | > loss_gen: 2.002492904663086 (1.9744223795668052) + | > loss_kl: 2.5089235305786133 (2.5035473308424008) + | > loss_feat: 3.997271776199341 (4.090280309645799) + | > loss_mel: 20.508058547973633 (20.514479358700935) + | > loss_duration: 1.3905104398727417 (1.3516383105820988) + | > amp_scaler: 128.0 (274.45255474452523) + | > loss_1: 30.407257080078125 (30.43436766366889) + | > grad_norm_1: tensor(443.9330, device='cuda:0') (tensor(325.4302, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 1.2006 (0.9229617597412891) + | > loader_time: 0.0207 (0.015152029312440082) + + + --> TIME: 2024-06-25 10:36:21 -- STEP: 573/591 -- GLOBAL_STEP: 28350 + | > loss_disc: 2.663076639175415 (2.6787238091906547) + | > loss_disc_real_0: 0.19154316186904907 (0.2016197939063658) + | > loss_disc_real_1: 0.2365189641714096 (0.2246206594439285) + | > loss_disc_real_2: 0.216170996427536 (0.2341392580387272) + | > loss_disc_real_3: 0.25753286480903625 (0.2341264308955241) + | > loss_disc_real_4: 0.2200527787208557 (0.2290334051140108) + | > loss_disc_real_5: 0.22601719200611115 (0.23894073935079324) + | > loss_0: 2.663076639175415 (2.6787238091906547) + | > grad_norm_0: tensor(11.9578, device='cuda:0') (tensor(10.9790, device='cuda:0')) + | > loss_gen: 1.985145926475525 (1.973679548990872) + | > loss_kl: 2.393345355987549 (2.5012051077205264) + | > loss_feat: 3.9543802738189697 (4.0800925324724595) + | > loss_mel: 20.650867462158203 (20.498377585286246) + | > loss_duration: 1.3732404708862305 (1.3532905751289916) + | > amp_scaler: 128.0 (268.06282722513043) + | > loss_1: 30.356979370117188 (30.40664532713033) + | > grad_norm_1: tensor(469.9282, device='cuda:0') (tensor(328.5695, device='cuda:0')) + | > current_lr_0: 0.00019882837179971516 + | > current_lr_1: 0.00019882837179971516 + | > step_time: 1.1682 (0.93508712034575) + | > loader_time: 0.0253 (0.015514753667888011) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0037143281165589676 (+0.0001362790452673077) + | > avg_loss_disc: 2.643645920652025 (-0.15320105755582736) + | > avg_loss_disc_real_0: 0.20365583230840398 (-0.1097195234070433) + | > avg_loss_disc_real_1: 0.19232257883599463 (-0.03314119926158418) + | > avg_loss_disc_real_2: 0.21764512138163788 (-0.019731352303890476) + | > avg_loss_disc_real_3: 0.21276232473393705 (-0.014607462160130757) + | > avg_loss_disc_real_4: 0.2095856840940232 (-0.04579423399681723) + | > avg_loss_disc_real_5: 0.2180473696678243 (-0.016551199745624606) + | > avg_loss_0: 2.643645920652025 (-0.15320105755582736) + | > avg_loss_gen: 1.8112427934687187 (-0.17260016025380898) + | > avg_loss_kl: 2.5347057707766267 (-0.030062442130229794) + | > avg_loss_feat: 3.970667270903893 (+0.09138023599665335) + | > avg_loss_mel: 20.44047879158182 (-0.23354010886334464) + | > avg_loss_duration: 1.363840437949972 (-0.0038303821644884994) + | > avg_loss_1: 30.120934993662733 (-0.34865306286101827) + + + > EPOCH: 49/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 10:36:51)  + + --> TIME: 2024-06-25 10:36:58 -- STEP: 7/591 -- GLOBAL_STEP: 28375 + | > loss_disc: 2.7232882976531982 (2.665200574057443) + | > loss_disc_real_0: 0.1729486584663391 (0.1966835515839713) + | > loss_disc_real_1: 0.1886598765850067 (0.22100346854754857) + | > loss_disc_real_2: 0.22510232031345367 (0.23089394399097987) + | > loss_disc_real_3: 0.18995939195156097 (0.23000283326421464) + | > loss_disc_real_4: 0.19188669323921204 (0.21907796604292734) + | > loss_disc_real_5: 0.21210378408432007 (0.24343211097376688) + | > loss_0: 2.7232882976531982 (2.665200574057443) + | > grad_norm_0: tensor(8.7857, device='cuda:0') (tensor(10.6423, device='cuda:0')) + | > loss_gen: 2.032778024673462 (2.0457992553710938) + | > loss_kl: 2.6311492919921875 (2.6791636603219167) + | > loss_feat: 4.258764743804932 (4.481863975524902) + | > loss_mel: 20.199853897094727 (20.57041331699916) + | > loss_duration: 1.3067185878753662 (1.3275116511753626) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.429264068603516 (31.104751314435685) + | > grad_norm_1: tensor(222.9583, device='cuda:0') (tensor(517.3679, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.7803 (0.7730236053466797) + | > loader_time: 0.0085 (0.008455821446010045) + + + --> TIME: 2024-06-25 10:37:18 -- STEP: 32/591 -- GLOBAL_STEP: 28400 + | > loss_disc: 2.714212417602539 (2.667819105088711) + | > loss_disc_real_0: 0.15891312062740326 (0.20168703282251954) + | > loss_disc_real_1: 0.18940608203411102 (0.2203594814054668) + | > loss_disc_real_2: 0.22947221994400024 (0.23475443152710795) + | > loss_disc_real_3: 0.2351342737674713 (0.2339360793121159) + | > loss_disc_real_4: 0.22213128209114075 (0.2278200094588101) + | > loss_disc_real_5: 0.24873995780944824 (0.24072647094726562) + | > loss_0: 2.714212417602539 (2.667819105088711) + | > grad_norm_0: tensor(8.3503, device='cuda:0') (tensor(11.1896, device='cuda:0')) + | > loss_gen: 2.0600075721740723 (2.0318151488900194) + | > loss_kl: 2.6057353019714355 (2.6349044963717456) + | > loss_feat: 4.175288677215576 (4.355394810438156) + | > loss_mel: 20.728565216064453 (20.62424749135971) + | > loss_duration: 1.327558994293213 (1.3169518373906615) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.89715576171875 (30.96331375837326) + | > grad_norm_1: tensor(351.3025, device='cuda:0') (tensor(563.4346, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.786 (0.7844126373529434) + | > loader_time: 0.0096 (0.009403489530086516) + + + --> TIME: 2024-06-25 10:37:39 -- STEP: 57/591 -- GLOBAL_STEP: 28425 + | > loss_disc: 2.6454479694366455 (2.6684116313332007) + | > loss_disc_real_0: 0.23491927981376648 (0.20215374126768948) + | > loss_disc_real_1: 0.2003006786108017 (0.22220458090305328) + | > loss_disc_real_2: 0.2603037655353546 (0.23536678834965355) + | > loss_disc_real_3: 0.2156447023153305 (0.2338682211804808) + | > loss_disc_real_4: 0.23629948496818542 (0.22922118635554062) + | > loss_disc_real_5: 0.24320809543132782 (0.239700651221108) + | > loss_0: 2.6454479694366455 (2.6684116313332007) + | > grad_norm_0: tensor(9.5023, device='cuda:0') (tensor(11.0996, device='cuda:0')) + | > loss_gen: 1.9296314716339111 (2.011833084257025) + | > loss_kl: 2.5880789756774902 (2.584880958523666) + | > loss_feat: 4.080186367034912 (4.312488187823378) + | > loss_mel: 20.12447738647461 (20.615754043846803) + | > loss_duration: 1.299890160560608 (1.3269381920496621) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.022266387939453 (30.85189447904888) + | > grad_norm_1: tensor(210.7108, device='cuda:0') (tensor(495.0839, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.8349 (0.7933755882999354) + | > loader_time: 0.0102 (0.009712834107248405) + + + --> TIME: 2024-06-25 10:38:00 -- STEP: 82/591 -- GLOBAL_STEP: 28450 + | > loss_disc: 2.6611814498901367 (2.6694548798770446) + | > loss_disc_real_0: 0.17613813281059265 (0.20158161913476338) + | > loss_disc_real_1: 0.21191632747650146 (0.22286248643223833) + | > loss_disc_real_2: 0.2043674737215042 (0.2338708419261909) + | > loss_disc_real_3: 0.26531046628952026 (0.23390780116726712) + | > loss_disc_real_4: 0.22440534830093384 (0.2287305911503187) + | > loss_disc_real_5: 0.24922199547290802 (0.2394592871026295) + | > loss_0: 2.6611814498901367 (2.6694548798770446) + | > grad_norm_0: tensor(6.8704, device='cuda:0') (tensor(11.3674, device='cuda:0')) + | > loss_gen: 2.1310462951660156 (2.003297062908731) + | > loss_kl: 2.546787977218628 (2.5716336645731106) + | > loss_feat: 4.387410640716553 (4.265737565552316) + | > loss_mel: 20.41130256652832 (20.593955226060825) + | > loss_duration: 1.3616297245025635 (1.3340131146151846) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.838176727294922 (30.768636610449814) + | > grad_norm_1: tensor(530.5370, device='cuda:0') (tensor(484.1350, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.8226 (0.800725602522129) + | > loader_time: 0.0118 (0.010085559472805114) + + + --> TIME: 2024-06-25 10:38:21 -- STEP: 107/591 -- GLOBAL_STEP: 28475 + | > loss_disc: 2.682772397994995 (2.6745513978405535) + | > loss_disc_real_0: 0.15970459580421448 (0.20223566207373253) + | > loss_disc_real_1: 0.22414346039295197 (0.22374655445602454) + | > loss_disc_real_2: 0.2459585815668106 (0.23600340502284398) + | > loss_disc_real_3: 0.2225755751132965 (0.23414389162420113) + | > loss_disc_real_4: 0.230199933052063 (0.22863451844063876) + | > loss_disc_real_5: 0.2543086111545563 (0.23899929573602766) + | > loss_0: 2.682772397994995 (2.6745513978405535) + | > grad_norm_0: tensor(9.4910, device='cuda:0') (tensor(11.8706, device='cuda:0')) + | > loss_gen: 1.884749412536621 (2.0014840085929797) + | > loss_kl: 2.597332239151001 (2.5588078186890777) + | > loss_feat: 4.056262969970703 (4.24592214210011) + | > loss_mel: 20.285388946533203 (20.615801641874228) + | > loss_duration: 1.3269647359848022 (1.339009575754682) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.150699615478516 (30.761025259427935) + | > grad_norm_1: tensor(297.6819, device='cuda:0') (tensor(496.7344, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.8322 (0.8075814581363001) + | > loader_time: 0.0108 (0.010256152286707797) + + + --> TIME: 2024-06-25 10:38:42 -- STEP: 132/591 -- GLOBAL_STEP: 28500 + | > loss_disc: 2.6568124294281006 (2.674802227453752) + | > loss_disc_real_0: 0.13586851954460144 (0.2017132779865554) + | > loss_disc_real_1: 0.21664723753929138 (0.22345145656303925) + | > loss_disc_real_2: 0.19886471331119537 (0.2357986948706887) + | > loss_disc_real_3: 0.2089824229478836 (0.23380169295000308) + | > loss_disc_real_4: 0.22313401103019714 (0.228767055227901) + | > loss_disc_real_5: 0.24710144102573395 (0.23913330866983443) + | > loss_0: 2.6568124294281006 (2.674802227453752) + | > grad_norm_0: tensor(11.6472, device='cuda:0') (tensor(11.8809, device='cuda:0')) + | > loss_gen: 2.173619270324707 (1.9991118962114502) + | > loss_kl: 2.613058090209961 (2.5521991523829364) + | > loss_feat: 4.255616664886475 (4.2391682231065) + | > loss_mel: 20.776540756225586 (20.630067796418164) + | > loss_duration: 1.3257657289505005 (1.3406461459217645) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.144601821899414 (30.76119328990127) + | > grad_norm_1: tensor(465.3002, device='cuda:0') (tensor(471.3141, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.8393 (0.8150445891149116) + | > loader_time: 0.0108 (0.010467747847239176) + + + --> TIME: 2024-06-25 10:39:04 -- STEP: 157/591 -- GLOBAL_STEP: 28525 + | > loss_disc: 2.7146477699279785 (2.679370580964786) + | > loss_disc_real_0: 0.18728423118591309 (0.20269907958758102) + | > loss_disc_real_1: 0.2109329104423523 (0.22410930417905192) + | > loss_disc_real_2: 0.24242515861988068 (0.2353836045523358) + | > loss_disc_real_3: 0.21030282974243164 (0.2334340804131927) + | > loss_disc_real_4: 0.20744313299655914 (0.22983368234649584) + | > loss_disc_real_5: 0.2131698727607727 (0.2392673942313832) + | > loss_0: 2.7146477699279785 (2.679370580964786) + | > grad_norm_0: tensor(4.6803, device='cuda:0') (tensor(12.0398, device='cuda:0')) + | > loss_gen: 1.9500254392623901 (1.9889437560063254) + | > loss_kl: 2.489621162414551 (2.5400254270833007) + | > loss_feat: 4.236617088317871 (4.216614805209409) + | > loss_mel: 19.97208595275879 (20.609901999212386) + | > loss_duration: 1.303323745727539 (1.3391083539671194) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.95167350769043 (30.69459441361154) + | > grad_norm_1: tensor(248.7321, device='cuda:0') (tensor(428.6277, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.859 (0.820830184183303) + | > loader_time: 0.0128 (0.010692991268862587) + + + --> TIME: 2024-06-25 10:39:26 -- STEP: 182/591 -- GLOBAL_STEP: 28550 + | > loss_disc: 2.663207769393921 (2.681328921527653) + | > loss_disc_real_0: 0.2147500067949295 (0.20295798094881756) + | > loss_disc_real_1: 0.28677496314048767 (0.22460496916875738) + | > loss_disc_real_2: 0.22299817204475403 (0.23520772347410956) + | > loss_disc_real_3: 0.24019403755664825 (0.23378002365211864) + | > loss_disc_real_4: 0.25654667615890503 (0.23049586544652562) + | > loss_disc_real_5: 0.2385486662387848 (0.23911166543161477) + | > loss_0: 2.663207769393921 (2.681328921527653) + | > grad_norm_0: tensor(6.2720, device='cuda:0') (tensor(11.4517, device='cuda:0')) + | > loss_gen: 1.9683555364608765 (1.9864136091955413) + | > loss_kl: 2.487847089767456 (2.5332195523020977) + | > loss_feat: 4.107373237609863 (4.208005833101797) + | > loss_mel: 20.30962562561035 (20.601135243426324) + | > loss_duration: 1.3312886953353882 (1.3375779167636408) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.204490661621094 (30.666352209154066) + | > grad_norm_1: tensor(157.3530, device='cuda:0') (tensor(398.5533, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.8511 (0.8267207302889981) + | > loader_time: 0.0118 (0.010892772412562117) + + + --> TIME: 2024-06-25 10:39:48 -- STEP: 207/591 -- GLOBAL_STEP: 28575 + | > loss_disc: 2.6827688217163086 (2.682438742135458) + | > loss_disc_real_0: 0.2650172710418701 (0.2033380626696319) + | > loss_disc_real_1: 0.2118828296661377 (0.22458056133726373) + | > loss_disc_real_2: 0.2228756844997406 (0.23527331355113337) + | > loss_disc_real_3: 0.23205621540546417 (0.23404990518150698) + | > loss_disc_real_4: 0.24640409648418427 (0.23071128957801396) + | > loss_disc_real_5: 0.2453472912311554 (0.23903610128999334) + | > loss_0: 2.6827688217163086 (2.682438742135458) + | > grad_norm_0: tensor(15.1674, device='cuda:0') (tensor(11.3254, device='cuda:0')) + | > loss_gen: 1.7918422222137451 (1.9839498795173016) + | > loss_kl: 2.512479066848755 (2.5294082683065664) + | > loss_feat: 3.8370938301086426 (4.1964781065493995) + | > loss_mel: 19.882511138916016 (20.597316060089273) + | > loss_duration: 1.376577377319336 (1.3372081852189581) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.400503158569336 (30.644360528475996) + | > grad_norm_1: tensor(211.1851, device='cuda:0') (tensor(381.0776, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.8624 (0.8322494018480974) + | > loader_time: 0.0122 (0.011119772270681786) + + + --> TIME: 2024-06-25 10:40:11 -- STEP: 232/591 -- GLOBAL_STEP: 28600 + | > loss_disc: 2.678182363510132 (2.6846859455108643) + | > loss_disc_real_0: 0.17085221409797668 (0.20366162912727426) + | > loss_disc_real_1: 0.22597083449363708 (0.22467582293882454) + | > loss_disc_real_2: 0.22421175241470337 (0.23530368025189843) + | > loss_disc_real_3: 0.24017135798931122 (0.23387316640081077) + | > loss_disc_real_4: 0.23251880705356598 (0.2309032877072178) + | > loss_disc_real_5: 0.2421974390745163 (0.2387304745357612) + | > loss_0: 2.678182363510132 (2.6846859455108643) + | > grad_norm_0: tensor(5.6067, device='cuda:0') (tensor(11.3818, device='cuda:0')) + | > loss_gen: 1.9654918909072876 (1.981793729909535) + | > loss_kl: 2.3727314472198486 (2.5271096733109704) + | > loss_feat: 4.3531107902526855 (4.18270075835031) + | > loss_mel: 20.646944046020508 (20.587029407764316) + | > loss_duration: 1.3595798015594482 (1.3370406617378363) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.697858810424805 (30.615674265499774) + | > grad_norm_1: tensor(188.9711, device='cuda:0') (tensor(349.9372, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.8913 (0.8374514661986252) + | > loader_time: 0.0144 (0.011433085490917346) + + + --> TIME: 2024-06-25 10:40:34 -- STEP: 257/591 -- GLOBAL_STEP: 28625 + | > loss_disc: 2.6549911499023438 (2.68618965705545) + | > loss_disc_real_0: 0.20896679162979126 (0.20365978855683178) + | > loss_disc_real_1: 0.23339758813381195 (0.22452054349596864) + | > loss_disc_real_2: 0.23833148181438446 (0.2356116693423416) + | > loss_disc_real_3: 0.2556127905845642 (0.23384882144427022) + | > loss_disc_real_4: 0.2261999249458313 (0.23135095469450673) + | > loss_disc_real_5: 0.21195200085639954 (0.2390095327150961) + | > loss_0: 2.6549911499023438 (2.68618965705545) + | > grad_norm_0: tensor(6.2754, device='cuda:0') (tensor(11.0169, device='cuda:0')) + | > loss_gen: 1.8769605159759521 (1.9766608133390253) + | > loss_kl: 2.519412040710449 (2.5251843688089095) + | > loss_feat: 4.018304347991943 (4.167707072157808) + | > loss_mel: 20.561115264892578 (20.565968702739326) + | > loss_duration: 1.333968162536621 (1.3376468777192703) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.30976104736328 (30.573167889962402) + | > grad_norm_1: tensor(112.9732, device='cuda:0') (tensor(325.2958, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.9168 (0.8450098046996715) + | > loader_time: 0.0154 (0.011771687273849316) + + + --> TIME: 2024-06-25 10:40:57 -- STEP: 282/591 -- GLOBAL_STEP: 28650 + | > loss_disc: 2.7415125370025635 (2.685656358164253) + | > loss_disc_real_0: 0.2957921624183655 (0.2033095982269192) + | > loss_disc_real_1: 0.2059941440820694 (0.22447244019795817) + | > loss_disc_real_2: 0.2054731249809265 (0.23543587062798493) + | > loss_disc_real_3: 0.19925712049007416 (0.23361474138202395) + | > loss_disc_real_4: 0.23210860788822174 (0.2316661213412352) + | > loss_disc_real_5: 0.24179822206497192 (0.23915849488677707) + | > loss_0: 2.7415125370025635 (2.685656358164253) + | > grad_norm_0: tensor(20.4143, device='cuda:0') (tensor(11.0092, device='cuda:0')) + | > loss_gen: 1.963055968284607 (1.9762208736534659) + | > loss_kl: 2.612088203430176 (2.5231286321125976) + | > loss_feat: 4.170956611633301 (4.164888179894038) + | > loss_mel: 20.258771896362305 (20.561324735059817) + | > loss_duration: 1.3562934398651123 (1.338252074752293) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.36116600036621 (30.563814541972274) + | > grad_norm_1: tensor(209.2246, device='cuda:0') (tensor(312.4769, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.8926 (0.8495163866814147) + | > loader_time: 0.0137 (0.012049215905209814) + + + --> TIME: 2024-06-25 10:41:20 -- STEP: 307/591 -- GLOBAL_STEP: 28675 + | > loss_disc: 2.6925084590911865 (2.6862312815476708) + | > loss_disc_real_0: 0.1820906549692154 (0.20271043887161663) + | > loss_disc_real_1: 0.1639048159122467 (0.22481665466623896) + | > loss_disc_real_2: 0.23428891599178314 (0.23529224141412916) + | > loss_disc_real_3: 0.2465973198413849 (0.23399330000923976) + | > loss_disc_real_4: 0.18844735622406006 (0.23149938235842052) + | > loss_disc_real_5: 0.22635960578918457 (0.23906246638259204) + | > loss_0: 2.6925084590911865 (2.6862312815476708) + | > grad_norm_0: tensor(6.0893, device='cuda:0') (tensor(10.9070, device='cuda:0')) + | > loss_gen: 1.9099962711334229 (1.973920443158973) + | > loss_kl: 2.465761423110962 (2.52411181996622) + | > loss_feat: 4.067358016967773 (4.1527224744181686) + | > loss_mel: 20.703577041625977 (20.553248986359147) + | > loss_duration: 1.364851951599121 (1.3391072920945255) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.51154327392578 (30.543111068029745) + | > grad_norm_1: tensor(80.6151, device='cuda:0') (tensor(312.7141, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.9172 (0.8547013318499835) + | > loader_time: 0.017 (0.012341967623086248) + + + --> TIME: 2024-06-25 10:41:44 -- STEP: 332/591 -- GLOBAL_STEP: 28700 + | > loss_disc: 2.645622491836548 (2.686077329049629) + | > loss_disc_real_0: 0.21351611614227295 (0.2029026971164956) + | > loss_disc_real_1: 0.20940272510051727 (0.22467907239873725) + | > loss_disc_real_2: 0.23568782210350037 (0.235239685703832) + | > loss_disc_real_3: 0.22644343972206116 (0.2337866617164698) + | > loss_disc_real_4: 0.2300829291343689 (0.23159667818122595) + | > loss_disc_real_5: 0.23666414618492126 (0.23921246068125748) + | > loss_0: 2.645622491836548 (2.686077329049629) + | > grad_norm_0: tensor(3.8616, device='cuda:0') (tensor(10.9334, device='cuda:0')) + | > loss_gen: 1.9184436798095703 (1.9725356407194252) + | > loss_kl: 2.4350876808166504 (2.5204667168927477) + | > loss_feat: 4.022439479827881 (4.145415248641052) + | > loss_mel: 20.608583450317383 (20.54807234959431) + | > loss_duration: 1.3647058010101318 (1.339800118681896) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.349260330200195 (30.526290106486126) + | > grad_norm_1: tensor(34.8555, device='cuda:0') (tensor(303.0672, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.9172 (0.8597486830619444) + | > loader_time: 0.0169 (0.012585602610944276) + + + --> TIME: 2024-06-25 10:42:08 -- STEP: 357/591 -- GLOBAL_STEP: 28725 + | > loss_disc: 2.6322693824768066 (2.686769285121887) + | > loss_disc_real_0: 0.17570538818836212 (0.2029073031551364) + | > loss_disc_real_1: 0.19801831245422363 (0.22482327699494295) + | > loss_disc_real_2: 0.2043406367301941 (0.2353954078937445) + | > loss_disc_real_3: 0.21166299283504486 (0.233772732839197) + | > loss_disc_real_4: 0.22110100090503693 (0.23170632541346614) + | > loss_disc_real_5: 0.23435616493225098 (0.23916027154408248) + | > loss_0: 2.6322693824768066 (2.686769285121887) + | > grad_norm_0: tensor(4.5430, device='cuda:0') (tensor(10.9516, device='cuda:0')) + | > loss_gen: 1.9401981830596924 (1.9711439963482342) + | > loss_kl: 2.4654135704040527 (2.5184335922326695) + | > loss_feat: 4.0730509757995605 (4.135744822793316) + | > loss_mel: 20.253337860107422 (20.5391918577734) + | > loss_duration: 1.3891103267669678 (1.3409707606339656) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.121110916137695 (30.50548504313835) + | > grad_norm_1: tensor(226.7364, device='cuda:0') (tensor(298.9111, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.9252 (0.8652708790883297) + | > loader_time: 0.0151 (0.01287157528874588) + + + --> TIME: 2024-06-25 10:42:32 -- STEP: 382/591 -- GLOBAL_STEP: 28750 + | > loss_disc: 2.613288164138794 (2.686759949354603) + | > loss_disc_real_0: 0.21064287424087524 (0.20283102675180162) + | > loss_disc_real_1: 0.19452758133411407 (0.22477174811650322) + | > loss_disc_real_2: 0.23244553804397583 (0.2352279052918494) + | > loss_disc_real_3: 0.21970945596694946 (0.2337334239904169) + | > loss_disc_real_4: 0.21905270218849182 (0.2317732239034787) + | > loss_disc_real_5: 0.2547720968723297 (0.23923359709884484) + | > loss_0: 2.613288164138794 (2.686759949354603) + | > grad_norm_0: tensor(6.3915, device='cuda:0') (tensor(10.9339, device='cuda:0')) + | > loss_gen: 2.015016555786133 (1.9697737172636063) + | > loss_kl: 2.5638656616210938 (2.514608115425909) + | > loss_feat: 4.123532772064209 (4.124659429670007) + | > loss_mel: 20.297208786010742 (20.525883514843706) + | > loss_duration: 1.3395435810089111 (1.342462028820478) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.339168548583984 (30.477386804151287) + | > grad_norm_1: tensor(526.9570, device='cuda:0') (tensor(297.0885, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.9624 (0.8709572503704051) + | > loader_time: 0.0155 (0.013153395727666894) + + + --> TIME: 2024-06-25 10:42:57 -- STEP: 407/591 -- GLOBAL_STEP: 28775 + | > loss_disc: 2.7106523513793945 (2.6876811893331936) + | > loss_disc_real_0: 0.22014665603637695 (0.2026919300601582) + | > loss_disc_real_1: 0.23642483353614807 (0.2250252330918277) + | > loss_disc_real_2: 0.2453242689371109 (0.23525599366470404) + | > loss_disc_real_3: 0.2322283685207367 (0.23390686446091466) + | > loss_disc_real_4: 0.24104642868041992 (0.2317092397435584) + | > loss_disc_real_5: 0.23972029983997345 (0.23921192970117713) + | > loss_0: 2.7106523513793945 (2.6876811893331936) + | > grad_norm_0: tensor(7.7972, device='cuda:0') (tensor(10.8361, device='cuda:0')) + | > loss_gen: 1.9025096893310547 (1.9687185914276273) + | > loss_kl: 2.366598606109619 (2.5112540792188724) + | > loss_feat: 4.023189544677734 (4.114974471802211) + | > loss_mel: 20.519725799560547 (20.519956396604364) + | > loss_duration: 1.3222236633300781 (1.3435494967116193) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.134246826171875 (30.458453044844493) + | > grad_norm_1: tensor(94.7878, device='cuda:0') (tensor(299.1403, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 0.986 (0.8769761578749673) + | > loader_time: 0.0187 (0.013441659308768616) + + + --> TIME: 2024-06-25 10:43:23 -- STEP: 432/591 -- GLOBAL_STEP: 28800 + | > loss_disc: 2.705552577972412 (2.6876987389944236) + | > loss_disc_real_0: 0.20354489982128143 (0.20252463243970717) + | > loss_disc_real_1: 0.2540331184864044 (0.2249963964784035) + | > loss_disc_real_2: 0.25910210609436035 (0.2353477836759002) + | > loss_disc_real_3: 0.2593197822570801 (0.2340060872612176) + | > loss_disc_real_4: 0.21324177086353302 (0.23168527200404138) + | > loss_disc_real_5: 0.23070724308490753 (0.23925423415170777) + | > loss_0: 2.705552577972412 (2.6876987389944236) + | > grad_norm_0: tensor(5.3046, device='cuda:0') (tensor(10.8458, device='cuda:0')) + | > loss_gen: 2.000713348388672 (1.9681402400687888) + | > loss_kl: 2.512840986251831 (2.511371997219546) + | > loss_feat: 3.6628408432006836 (4.106389763178649) + | > loss_mel: 19.12873077392578 (20.507497610869247) + | > loss_duration: 1.3415148258209229 (1.3445096098714413) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 28.64664077758789 (30.437909219000076) + | > grad_norm_1: tensor(384.3799, device='cuda:0') (tensor(299.8348, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 1.0379 (0.8836117254363166) + | > loader_time: 0.0196 (0.013711513192565361) + + + --> TIME: 2024-06-25 10:43:49 -- STEP: 457/591 -- GLOBAL_STEP: 28825 + | > loss_disc: 2.709773540496826 (2.6869422403414887) + | > loss_disc_real_0: 0.18872526288032532 (0.20249472640014582) + | > loss_disc_real_1: 0.2800392806529999 (0.22506275329600278) + | > loss_disc_real_2: 0.23561684787273407 (0.23523289995589977) + | > loss_disc_real_3: 0.23178809881210327 (0.23393956081089878) + | > loss_disc_real_4: 0.22604157030582428 (0.23144942985060957) + | > loss_disc_real_5: 0.25502246618270874 (0.23926347464779535) + | > loss_0: 2.709773540496826 (2.6869422403414887) + | > grad_norm_0: tensor(9.9138, device='cuda:0') (tensor(10.8819, device='cuda:0')) + | > loss_gen: 1.899071216583252 (1.9681014644238828) + | > loss_kl: 2.5162601470947266 (2.5081086842184264) + | > loss_feat: 3.6682822704315186 (4.099027784961057) + | > loss_mel: 20.10868263244629 (20.49793020119032) + | > loss_duration: 1.343641757965088 (1.3454838479895568) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.53593635559082 (30.41865197313171) + | > grad_norm_1: tensor(424.2248, device='cuda:0') (tensor(306.7661, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 1.0018 (0.890960994382395) + | > loader_time: 0.0158 (0.013993586700906908) + + + --> TIME: 2024-06-25 10:44:15 -- STEP: 482/591 -- GLOBAL_STEP: 28850 + | > loss_disc: 2.7368531227111816 (2.687958663429958) + | > loss_disc_real_0: 0.2137274146080017 (0.20318369103665174) + | > loss_disc_real_1: 0.2885405719280243 (0.22512294568461502) + | > loss_disc_real_2: 0.25070732831954956 (0.23527582057778765) + | > loss_disc_real_3: 0.26865530014038086 (0.23395531905884565) + | > loss_disc_real_4: 0.2231239527463913 (0.23096112489576653) + | > loss_disc_real_5: 0.2632034420967102 (0.23920676266751348) + | > loss_0: 2.7368531227111816 (2.687958663429958) + | > grad_norm_0: tensor(7.6662, device='cuda:0') (tensor(11.2181, device='cuda:0')) + | > loss_gen: 1.9765067100524902 (1.9685893179964722) + | > loss_kl: 2.448479175567627 (2.5045614252446606) + | > loss_feat: 3.5459084510803223 (4.088663833764576) + | > loss_mel: 19.741147994995117 (20.482362482063024) + | > loss_duration: 1.3596148490905762 (1.3467976829323043) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.071657180786133 (30.390974725430436) + | > grad_norm_1: tensor(351.9689, device='cuda:0') (tensor(319.2271, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 1.0397 (0.8985218541750769) + | > loader_time: 0.0195 (0.014284779422016072) + + + --> TIME: 2024-06-25 10:44:43 -- STEP: 507/591 -- GLOBAL_STEP: 28875 + | > loss_disc: 2.7146975994110107 (2.6875831986790355) + | > loss_disc_real_0: 0.2080635130405426 (0.20310582629087173) + | > loss_disc_real_1: 0.2597055435180664 (0.22507154677041183) + | > loss_disc_real_2: 0.23060154914855957 (0.23518948768370251) + | > loss_disc_real_3: 0.23585852980613708 (0.2340457380346998) + | > loss_disc_real_4: 0.24736201763153076 (0.23073644800886833) + | > loss_disc_real_5: 0.2597949206829071 (0.23922391227011144) + | > loss_0: 2.7146975994110107 (2.6875831986790355) + | > grad_norm_0: tensor(7.2342, device='cuda:0') (tensor(11.1348, device='cuda:0')) + | > loss_gen: 1.9306875467300415 (1.968446105659831) + | > loss_kl: 2.552138090133667 (2.504386853419348) + | > loss_feat: 3.9460368156433105 (4.082368083724372) + | > loss_mel: 19.9627685546875 (20.46852622323724) + | > loss_duration: 1.4092795848846436 (1.34830240008864) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.8009090423584 (30.372029641205977) + | > grad_norm_1: tensor(149.1680, device='cuda:0') (tensor(325.8932, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 1.0741 (0.9075251533199814) + | > loader_time: 0.0224 (0.01461493193045171) + + + --> TIME: 2024-06-25 10:45:11 -- STEP: 532/591 -- GLOBAL_STEP: 28900 + | > loss_disc: 2.684887170791626 (2.688245741048255) + | > loss_disc_real_0: 0.23478484153747559 (0.20317421660275387) + | > loss_disc_real_1: 0.21669597923755646 (0.22510504131590514) + | > loss_disc_real_2: 0.20708179473876953 (0.23521841309012323) + | > loss_disc_real_3: 0.20589283108711243 (0.23407419353611483) + | > loss_disc_real_4: 0.22759000957012177 (0.23078502493357297) + | > loss_disc_real_5: 0.21280992031097412 (0.23916318808171086) + | > loss_0: 2.684887170791626 (2.688245741048255) + | > grad_norm_0: tensor(8.7408, device='cuda:0') (tensor(11.1966, device='cuda:0')) + | > loss_gen: 1.8490105867385864 (1.9665648475625461) + | > loss_kl: 2.5283031463623047 (2.5027477190010536) + | > loss_feat: 3.7615251541137695 (4.071391305080929) + | > loss_mel: 20.331432342529297 (20.453277225781218) + | > loss_duration: 1.3853163719177246 (1.3499575508268262) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.855588912963867 (30.3439386088149) + | > grad_norm_1: tensor(380.9084, device='cuda:0') (tensor(326.3664, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 1.0822 (0.9175736693511332) + | > loader_time: 0.023 (0.014946369748366514) + + + --> TIME: 2024-06-25 10:45:42 -- STEP: 557/591 -- GLOBAL_STEP: 28925 + | > loss_disc: 2.7158219814300537 (2.6880011806796236) + | > loss_disc_real_0: 0.18857258558273315 (0.20293012042019914) + | > loss_disc_real_1: 0.2472665160894394 (0.2250264440822002) + | > loss_disc_real_2: 0.21658265590667725 (0.23510853793719508) + | > loss_disc_real_3: 0.23056332767009735 (0.2341608828873352) + | > loss_disc_real_4: 0.2186175286769867 (0.23090458264042696) + | > loss_disc_real_5: 0.23275381326675415 (0.23919848699115978) + | > loss_0: 2.7158219814300537 (2.6880011806796236) + | > grad_norm_0: tensor(5.9818, device='cuda:0') (tensor(11.0749, device='cuda:0')) + | > loss_gen: 1.8881449699401855 (1.9655556858549) + | > loss_kl: 2.4260382652282715 (2.5011195485853) + | > loss_feat: 4.007377624511719 (4.060366098927852) + | > loss_mel: 19.859724044799805 (20.43800923126605) + | > loss_duration: 1.3793022632598877 (1.351628325999961) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.560588836669922 (30.31667886388152) + | > grad_norm_1: tensor(249.1079, device='cuda:0') (tensor(325.8357, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 1.2916 (0.930241254758578) + | > loader_time: 0.0245 (0.01528359443965892) + + + --> TIME: 2024-06-25 10:46:15 -- STEP: 582/591 -- GLOBAL_STEP: 28950 + | > loss_disc: 2.701145887374878 (2.687812527430428) + | > loss_disc_real_0: 0.18159864842891693 (0.2028609246425203) + | > loss_disc_real_1: 0.2341984510421753 (0.22501169646933317) + | > loss_disc_real_2: 0.24500635266304016 (0.23508973063779445) + | > loss_disc_real_3: 0.20773853361606598 (0.2341491482972689) + | > loss_disc_real_4: 0.2472696155309677 (0.2310302544039549) + | > loss_disc_real_5: 0.24374128878116608 (0.23920959812873827) + | > loss_0: 2.701145887374878 (2.687812527430428) + | > grad_norm_0: tensor(5.8760, device='cuda:0') (tensor(11.0218, device='cuda:0')) + | > loss_gen: 1.9795360565185547 (1.9647848175973008) + | > loss_kl: 2.470813035964966 (2.499213525929403) + | > loss_feat: 3.8053359985351562 (4.05253938792907) + | > loss_mel: 20.391565322875977 (20.423537450967384) + | > loss_duration: 1.376539707183838 (1.3532976125114158) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.023788452148438 (30.293372780187024) + | > grad_norm_1: tensor(98.3136, device='cuda:0') (tensor(318.6181, device='cuda:0')) + | > current_lr_0: 0.00019880351825324018 + | > current_lr_1: 0.00019880351825324018 + | > step_time: 1.2789 (0.9447223968112591) + | > loader_time: 0.0256 (0.015700016644402473) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003610737780307202 (-0.00010359033625176577) + | > avg_loss_disc: 2.7143150491917374 (+0.0706691285397123) + | > avg_loss_disc_real_0: 0.26264055263488845 (+0.05898472032648447) + | > avg_loss_disc_real_1: 0.19301542956778345 (+0.0006928507317888188) + | > avg_loss_disc_real_2: 0.22187278847745123 (+0.004227667095813348) + | > avg_loss_disc_real_3: 0.22834336091863347 (+0.015581036184696417) + | > avg_loss_disc_real_4: 0.23586125862091145 (+0.02627557452688825) + | > avg_loss_disc_real_5: 0.22916706952642887 (+0.011119699858604587) + | > avg_loss_0: 2.7143150491917374 (+0.0706691285397123) + | > avg_loss_gen: 1.8886532124052657 (+0.07741041893654699) + | > avg_loss_kl: 2.535844402110323 (+0.0011386313336960718) + | > avg_loss_feat: 3.8478180449059667 (-0.12284922599792614) + | > avg_loss_mel: 20.54223486717711 (+0.10175607559528999) + | > avg_loss_duration: 1.3607892279929303 (-0.0030512099570416673) + | > avg_loss_1: 30.175339759664332 (+0.054404766001599825) + + + > EPOCH: 50/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 10:46:33)  + + --> TIME: 2024-06-25 10:46:48 -- STEP: 16/591 -- GLOBAL_STEP: 28975 + | > loss_disc: 2.737009048461914 (2.6809221357107162) + | > loss_disc_real_0: 0.19385099411010742 (0.20077709015458822) + | > loss_disc_real_1: 0.19163189828395844 (0.2239725012332201) + | > loss_disc_real_2: 0.2713446617126465 (0.23469893634319305) + | > loss_disc_real_3: 0.2516235113143921 (0.23914095759391785) + | > loss_disc_real_4: 0.25361108779907227 (0.23157219123095274) + | > loss_disc_real_5: 0.24754080176353455 (0.2399943843483925) + | > loss_0: 2.737009048461914 (2.6809221357107162) + | > grad_norm_0: tensor(6.7465, device='cuda:0') (tensor(11.5147, device='cuda:0')) + | > loss_gen: 1.9890961647033691 (2.0032888427376747) + | > loss_kl: 2.5481574535369873 (2.5666698813438416) + | > loss_feat: 4.1757965087890625 (4.315983593463898) + | > loss_mel: 20.34649658203125 (20.68490993976593) + | > loss_duration: 1.3706564903259277 (1.3177137300372124) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.43020248413086 (30.88856601715088) + | > grad_norm_1: tensor(68.7489, device='cuda:0') (tensor(202.9765, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 1.0103 (0.786769300699234) + | > loader_time: 0.009 (0.008774608373641968) + + + --> TIME: 2024-06-25 10:47:08 -- STEP: 41/591 -- GLOBAL_STEP: 29000 + | > loss_disc: 2.693002939224243 (2.6883711931182117) + | > loss_disc_real_0: 0.22401276230812073 (0.20218039867354604) + | > loss_disc_real_1: 0.21348722279071808 (0.22511164662314626) + | > loss_disc_real_2: 0.2282358556985855 (0.2325620800256729) + | > loss_disc_real_3: 0.19722484052181244 (0.23581339709642457) + | > loss_disc_real_4: 0.22582405805587769 (0.2340940593219385) + | > loss_disc_real_5: 0.2442874163389206 (0.2404140277606685) + | > loss_0: 2.693002939224243 (2.6883711931182117) + | > grad_norm_0: tensor(6.5850, device='cuda:0') (tensor(9.4279, device='cuda:0')) + | > loss_gen: 1.8839048147201538 (1.9667700848928311) + | > loss_kl: 2.582188129425049 (2.5862951685742632) + | > loss_feat: 4.034060001373291 (4.227394208675477) + | > loss_mel: 20.149486541748047 (20.52056400950362) + | > loss_duration: 1.374871850013733 (1.3217190794828462) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.024513244628906 (30.622742676153415) + | > grad_norm_1: tensor(41.5685, device='cuda:0') (tensor(173.0290, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.7998 (0.791823224323552) + | > loader_time: 0.0108 (0.009332523113343774) + + + --> TIME: 2024-06-25 10:47:29 -- STEP: 66/591 -- GLOBAL_STEP: 29025 + | > loss_disc: 2.731398582458496 (2.683397816889213) + | > loss_disc_real_0: 0.25711363554000854 (0.20304563889900842) + | > loss_disc_real_1: 0.19070768356323242 (0.22421296776244135) + | > loss_disc_real_2: 0.23006802797317505 (0.2332750768824057) + | > loss_disc_real_3: 0.25264912843704224 (0.2353931970217011) + | > loss_disc_real_4: 0.2766607403755188 (0.23323894201806097) + | > loss_disc_real_5: 0.2542414367198944 (0.23876853422685104) + | > loss_0: 2.731398582458496 (2.683397816889213) + | > grad_norm_0: tensor(11.2680, device='cuda:0') (tensor(9.6786, device='cuda:0')) + | > loss_gen: 1.8542096614837646 (1.9766001087246519) + | > loss_kl: 2.5160140991210938 (2.5698958996570473) + | > loss_feat: 4.099661827087402 (4.226271499286997) + | > loss_mel: 19.807209014892578 (20.56280433770382) + | > loss_duration: 1.3340178728103638 (1.3293370351646885) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.611114501953125 (30.664908842606977) + | > grad_norm_1: tensor(92.5371, device='cuda:0') (tensor(215.9397, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.814 (0.798715504732999) + | > loader_time: 0.0101 (0.009632807789426864) + + + --> TIME: 2024-06-25 10:47:50 -- STEP: 91/591 -- GLOBAL_STEP: 29050 + | > loss_disc: 2.581968307495117 (2.6812659148331512) + | > loss_disc_real_0: 0.1330731362104416 (0.20276897973739183) + | > loss_disc_real_1: 0.19391225278377533 (0.2234479159444243) + | > loss_disc_real_2: 0.21238940954208374 (0.2319167890064009) + | > loss_disc_real_3: 0.20376217365264893 (0.23472602911047882) + | > loss_disc_real_4: 0.25076553225517273 (0.2327354691185794) + | > loss_disc_real_5: 0.26048287749290466 (0.23923692614822598) + | > loss_0: 2.581968307495117 (2.6812659148331512) + | > grad_norm_0: tensor(12.9469, device='cuda:0') (tensor(10.6065, device='cuda:0')) + | > loss_gen: 2.1987500190734863 (1.9748834646665132) + | > loss_kl: 2.560805559158325 (2.569238851358602) + | > loss_feat: 4.293548107147217 (4.200198794459247) + | > loss_mel: 20.73717498779297 (20.564087165580997) + | > loss_duration: 1.377117395401001 (1.336727907369425) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.167396545410156 (30.645136068155477) + | > grad_norm_1: tensor(607.9372, device='cuda:0') (tensor(245.7223, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.8276 (0.8060379971514692) + | > loader_time: 0.0111 (0.009924351514040777) + + + --> TIME: 2024-06-25 10:48:11 -- STEP: 116/591 -- GLOBAL_STEP: 29075 + | > loss_disc: 2.4693872928619385 (2.670758179549512) + | > loss_disc_real_0: 0.16770628094673157 (0.20149758721476999) + | > loss_disc_real_1: 0.24415187537670135 (0.22456791251897812) + | > loss_disc_real_2: 0.2380838245153427 (0.23256928358098555) + | > loss_disc_real_3: 0.2287651151418686 (0.23382666193205734) + | > loss_disc_real_4: 0.16645552217960358 (0.22794159810090886) + | > loss_disc_real_5: 0.21787725389003754 (0.23852288645916972) + | > loss_0: 2.4693872928619385 (2.670758179549512) + | > grad_norm_0: tensor(11.9828, device='cuda:0') (tensor(11.6465, device='cuda:0')) + | > loss_gen: 2.285414934158325 (1.9983773498699582) + | > loss_kl: 2.568081855773926 (2.557291193255063) + | > loss_feat: 4.928513526916504 (4.243009450106782) + | > loss_mel: 21.17612648010254 (20.578374402276403) + | > loss_duration: 1.3506808280944824 (1.3392783382843279) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.30881881713867 (30.716330610472582) + | > grad_norm_1: tensor(943.0678, device='cuda:0') (tensor(350.6783, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.8406 (0.8124867019982174) + | > loader_time: 0.0107 (0.010164848689375259) + + + --> TIME: 2024-06-25 10:48:32 -- STEP: 141/591 -- GLOBAL_STEP: 29100 + | > loss_disc: 2.7210073471069336 (2.669073164040314) + | > loss_disc_real_0: 0.26474112272262573 (0.20083453168048926) + | > loss_disc_real_1: 0.2182515412569046 (0.22410321869748706) + | > loss_disc_real_2: 0.24663159251213074 (0.23286617055852363) + | > loss_disc_real_3: 0.21205119788646698 (0.23366870525035452) + | > loss_disc_real_4: 0.23775658011436462 (0.22753474930076734) + | > loss_disc_real_5: 0.2338368147611618 (0.2391170066721896) + | > loss_0: 2.7210073471069336 (2.669073164040314) + | > grad_norm_0: tensor(10.0193, device='cuda:0') (tensor(11.4717, device='cuda:0')) + | > loss_gen: 1.8785779476165771 (2.003017017181883) + | > loss_kl: 2.507979154586792 (2.547828107860917) + | > loss_feat: 4.035414218902588 (4.251509661370132) + | > loss_mel: 20.275732040405273 (20.600411962955555) + | > loss_duration: 1.3090578317642212 (1.3395443495283734) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.00676155090332 (30.742311085369572) + | > grad_norm_1: tensor(146.7668, device='cuda:0') (tensor(385.5314, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.8515 (0.8180370246264952) + | > loader_time: 0.0109 (0.010428430340814258) + + + --> TIME: 2024-06-25 10:48:54 -- STEP: 166/591 -- GLOBAL_STEP: 29125 + | > loss_disc: 2.612696647644043 (2.6685669881751735) + | > loss_disc_real_0: 0.1702306717634201 (0.20066302974360534) + | > loss_disc_real_1: 0.21965932846069336 (0.2242951750396246) + | > loss_disc_real_2: 0.18622227013111115 (0.2325681003640933) + | > loss_disc_real_3: 0.22418151795864105 (0.23371714165052734) + | > loss_disc_real_4: 0.19532380998134613 (0.2268551996493914) + | > loss_disc_real_5: 0.21316106617450714 (0.23905120116759496) + | > loss_0: 2.612696647644043 (2.6685669881751735) + | > grad_norm_0: tensor(5.2681, device='cuda:0') (tensor(11.9286, device='cuda:0')) + | > loss_gen: 1.9971752166748047 (2.00408538206514) + | > loss_kl: 2.50323486328125 (2.536502986069184) + | > loss_feat: 4.217597007751465 (4.2512671114450455) + | > loss_mel: 20.537960052490234 (20.619125906243372) + | > loss_duration: 1.3472013473510742 (1.3384492928723257) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.603168487548828 (30.74943070239331) + | > grad_norm_1: tensor(176.3913, device='cuda:0') (tensor(406.5410, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.8472 (0.824267325631107) + | > loader_time: 0.0137 (0.010741847107209357) + + + --> TIME: 2024-06-25 10:49:16 -- STEP: 191/591 -- GLOBAL_STEP: 29150 + | > loss_disc: 2.5851259231567383 (2.6675667013797457) + | > loss_disc_real_0: 0.15017926692962646 (0.20072887771572742) + | > loss_disc_real_1: 0.20146875083446503 (0.22352906515460988) + | > loss_disc_real_2: 0.22609546780586243 (0.23332028137763758) + | > loss_disc_real_3: 0.20012538135051727 (0.23384578252962124) + | > loss_disc_real_4: 0.22850705683231354 (0.2269017608221913) + | > loss_disc_real_5: 0.2632787525653839 (0.2390061200289202) + | > loss_0: 2.5851259231567383 (2.6675667013797457) + | > grad_norm_0: tensor(15.4883, device='cuda:0') (tensor(12.0219, device='cuda:0')) + | > loss_gen: 2.2565293312072754 (2.006692097449177) + | > loss_kl: 2.6475541591644287 (2.5314395078189715) + | > loss_feat: 4.402294635772705 (4.255391130896758) + | > loss_mel: 20.712682723999023 (20.61486619679715) + | > loss_duration: 1.329238772392273 (1.336973493635967) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.348299026489258 (30.745362416611915) + | > grad_norm_1: tensor(761.7890, device='cuda:0') (tensor(429.5970, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.8845 (0.830140440875947) + | > loader_time: 0.0123 (0.010977070993153837) + + + --> TIME: 2024-06-25 10:49:39 -- STEP: 216/591 -- GLOBAL_STEP: 29175 + | > loss_disc: 2.744403839111328 (2.666808857961937) + | > loss_disc_real_0: 0.12820249795913696 (0.20068309576837) + | > loss_disc_real_1: 0.22712865471839905 (0.2234669627138862) + | > loss_disc_real_2: 0.2452867329120636 (0.23320535528990957) + | > loss_disc_real_3: 0.24528972804546356 (0.2339615859091282) + | > loss_disc_real_4: 0.2340068519115448 (0.22673912711993413) + | > loss_disc_real_5: 0.2681032121181488 (0.23878502431843016) + | > loss_0: 2.744403839111328 (2.666808857961937) + | > grad_norm_0: tensor(20.2875, device='cuda:0') (tensor(11.8231, device='cuda:0')) + | > loss_gen: 2.015509843826294 (2.0070330357110047) + | > loss_kl: 2.548119306564331 (2.526358336210251) + | > loss_feat: 4.0446624755859375 (4.256617782292536) + | > loss_mel: 20.23752784729004 (20.605082856284245) + | > loss_duration: 1.333552360534668 (1.3364157036498747) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.17937469482422 (30.731507742846453) + | > grad_norm_1: tensor(57.0613, device='cuda:0') (tensor(424.8628, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.8656 (0.83522030711174) + | > loader_time: 0.0129 (0.01122077747627541) + + + --> TIME: 2024-06-25 10:50:01 -- STEP: 241/591 -- GLOBAL_STEP: 29200 + | > loss_disc: 2.6915504932403564 (2.667711350927709) + | > loss_disc_real_0: 0.10758668184280396 (0.20065411705817424) + | > loss_disc_real_1: 0.2295086681842804 (0.22336465584044635) + | > loss_disc_real_2: 0.20837736129760742 (0.2332675863227409) + | > loss_disc_real_3: 0.22878527641296387 (0.23376165583915234) + | > loss_disc_real_4: 0.2653903663158417 (0.2276080640272481) + | > loss_disc_real_5: 0.2465289682149887 (0.23884449280891182) + | > loss_0: 2.6915504932403564 (2.667711350927709) + | > grad_norm_0: tensor(22.4671, device='cuda:0') (tensor(11.7698, device='cuda:0')) + | > loss_gen: 2.1425304412841797 (2.003547604153263) + | > loss_kl: 2.6641128063201904 (2.528902322919537) + | > loss_feat: 4.0748419761657715 (4.2444315154522725) + | > loss_mel: 20.477399826049805 (20.595100687747177) + | > loss_duration: 1.3577632904052734 (1.3359679433814722) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.71664810180664 (30.707950101353816) + | > grad_norm_1: tensor(76.6589, device='cuda:0') (tensor(394.1657, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.8954 (0.840271561966892) + | > loader_time: 0.0154 (0.011563646348185559) + + + --> TIME: 2024-06-25 10:50:24 -- STEP: 266/591 -- GLOBAL_STEP: 29225 + | > loss_disc: 2.672240734100342 (2.670733079874426) + | > loss_disc_real_0: 0.19196748733520508 (0.20083901821110484) + | > loss_disc_real_1: 0.24059562385082245 (0.22363989542525514) + | > loss_disc_real_2: 0.23250889778137207 (0.23367352464369365) + | > loss_disc_real_3: 0.2567150890827179 (0.2339287052925368) + | > loss_disc_real_4: 0.207020565867424 (0.22838962498240004) + | > loss_disc_real_5: 0.22076937556266785 (0.23882892419745153) + | > loss_0: 2.672240734100342 (2.670733079874426) + | > grad_norm_0: tensor(8.2947, device='cuda:0') (tensor(11.7228, device='cuda:0')) + | > loss_gen: 1.9842853546142578 (1.9989652033138976) + | > loss_kl: 2.4329023361206055 (2.5298163854986204) + | > loss_feat: 4.155974864959717 (4.22760875691148) + | > loss_mel: 20.460405349731445 (20.579370477145773) + | > loss_duration: 1.3416309356689453 (1.336052439266578) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.375198364257812 (30.67181328364781) + | > grad_norm_1: tensor(178.9646, device='cuda:0') (tensor(369.6426, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.9269 (0.8459433048291313) + | > loader_time: 0.015 (0.011843304885061164) + + + --> TIME: 2024-06-25 10:50:47 -- STEP: 291/591 -- GLOBAL_STEP: 29250 + | > loss_disc: 2.577836513519287 (2.670001247084837) + | > loss_disc_real_0: 0.25005048513412476 (0.200976241879242) + | > loss_disc_real_1: 0.2465064525604248 (0.22360436869241124) + | > loss_disc_real_2: 0.2313191145658493 (0.23352928142162532) + | > loss_disc_real_3: 0.21614405512809753 (0.23374074021565547) + | > loss_disc_real_4: 0.17096705734729767 (0.22818995126334252) + | > loss_disc_real_5: 0.2192031741142273 (0.23864265211259378) + | > loss_0: 2.577836513519287 (2.670001247084837) + | > grad_norm_0: tensor(12.9720, device='cuda:0') (tensor(11.4710, device='cuda:0')) + | > loss_gen: 2.086266279220581 (1.9973639081843515) + | > loss_kl: 2.524139642715454 (2.5280808157117916) + | > loss_feat: 4.432113170623779 (4.221160004638721) + | > loss_mel: 20.26375961303711 (20.56872461587702) + | > loss_duration: 1.3660129308700562 (1.33712649140571) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.672290802001953 (30.6524558509748) + | > grad_norm_1: tensor(674.7742, device='cuda:0') (tensor(369.0180, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.9576 (0.8510816957532742) + | > loader_time: 0.0153 (0.01212126856407349) + + + --> TIME: 2024-06-25 10:51:11 -- STEP: 316/591 -- GLOBAL_STEP: 29275 + | > loss_disc: 2.7394208908081055 (2.6696709741519964) + | > loss_disc_real_0: 0.26918473839759827 (0.20083458386833156) + | > loss_disc_real_1: 0.26415202021598816 (0.22380463783688187) + | > loss_disc_real_2: 0.2533324062824249 (0.23358409856505033) + | > loss_disc_real_3: 0.20511284470558167 (0.23374709180450137) + | > loss_disc_real_4: 0.2177128791809082 (0.2279232507453689) + | > loss_disc_real_5: 0.2359360009431839 (0.23862600326538086) + | > loss_0: 2.7394208908081055 (2.6696709741519964) + | > grad_norm_0: tensor(16.9355, device='cuda:0') (tensor(11.7030, device='cuda:0')) + | > loss_gen: 1.9634296894073486 (1.9989256892777683) + | > loss_kl: 2.5311384201049805 (2.5283909613573092) + | > loss_feat: 3.642852544784546 (4.216145569010622) + | > loss_mel: 20.422842025756836 (20.573082579842087) + | > loss_duration: 1.366248607635498 (1.3378185418587698) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.926511764526367 (30.654363366621958) + | > grad_norm_1: tensor(569.7193, device='cuda:0') (tensor(385.1177, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.9 (0.8569679486600655) + | > loader_time: 0.0141 (0.012362359445306316) + + + --> TIME: 2024-06-25 10:51:35 -- STEP: 341/591 -- GLOBAL_STEP: 29300 + | > loss_disc: 2.6571571826934814 (2.6694604833105084) + | > loss_disc_real_0: 0.2767956256866455 (0.20113019848411734) + | > loss_disc_real_1: 0.16825249791145325 (0.22353535313760084) + | > loss_disc_real_2: 0.20563502609729767 (0.23379748299324618) + | > loss_disc_real_3: 0.19855429232120514 (0.23414112334726842) + | > loss_disc_real_4: 0.1650778353214264 (0.22708984960384032) + | > loss_disc_real_5: 0.2297721654176712 (0.2386927278248096) + | > loss_0: 2.6571571826934814 (2.6694604833105084) + | > grad_norm_0: tensor(24.8752, device='cuda:0') (tensor(12.1092, device='cuda:0')) + | > loss_gen: 1.9656727313995361 (2.000752790232906) + | > loss_kl: 2.526644706726074 (2.526140336766619) + | > loss_feat: 4.13356351852417 (4.217521598262169) + | > loss_mel: 20.149518966674805 (20.56946285728833) + | > loss_duration: 1.351959228515625 (1.3391657595760318) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.12735939025879 (30.6530433554104) + | > grad_norm_1: tensor(811.0873, device='cuda:0') (tensor(408.0246, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.9328 (0.8626932419639874) + | > loader_time: 0.0157 (0.012623182839312509) + + + --> TIME: 2024-06-25 10:51:59 -- STEP: 366/591 -- GLOBAL_STEP: 29325 + | > loss_disc: 2.6163787841796875 (2.666822414580587) + | > loss_disc_real_0: 0.2463906854391098 (0.20063605593120465) + | > loss_disc_real_1: 0.19200044870376587 (0.2237822960634701) + | > loss_disc_real_2: 0.24518033862113953 (0.2337758247562445) + | > loss_disc_real_3: 0.22604051232337952 (0.23407299755538097) + | > loss_disc_real_4: 0.22821685671806335 (0.22660559664169946) + | > loss_disc_real_5: 0.24129551649093628 (0.23828959310315345) + | > loss_0: 2.6163787841796875 (2.666822414580587) + | > grad_norm_0: tensor(13.7672, device='cuda:0') (tensor(12.1039, device='cuda:0')) + | > loss_gen: 1.9256353378295898 (2.001647789947321) + | > loss_kl: 2.4692091941833496 (2.5234083569114962) + | > loss_feat: 4.110232353210449 (4.213060526248536) + | > loss_mel: 20.45769691467285 (20.558734904221485) + | > loss_duration: 1.386202096939087 (1.3406971119792084) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.348974227905273 (30.63754867074268) + | > grad_norm_1: tensor(609.3987, device='cuda:0') (tensor(422.4286, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.9551 (0.8688460536341847) + | > loader_time: 0.0157 (0.012875039720795843) + + + --> TIME: 2024-06-25 10:52:24 -- STEP: 391/591 -- GLOBAL_STEP: 29350 + | > loss_disc: 2.628941535949707 (2.667644110482059) + | > loss_disc_real_0: 0.19136516749858856 (0.20051043332004181) + | > loss_disc_real_1: 0.23034663498401642 (0.22390495713257122) + | > loss_disc_real_2: 0.2675512433052063 (0.23392840716844934) + | > loss_disc_real_3: 0.1825629472732544 (0.234021320443629) + | > loss_disc_real_4: 0.23337297141551971 (0.22696886224972318) + | > loss_disc_real_5: 0.25943416357040405 (0.23836834218038622) + | > loss_0: 2.628941535949707 (2.667644110482059) + | > grad_norm_0: tensor(7.8469, device='cuda:0') (tensor(12.0354, device='cuda:0')) + | > loss_gen: 1.9557644128799438 (2.000525481560649) + | > loss_kl: 2.4592010974884033 (2.5213897368487164) + | > loss_feat: 4.106714248657227 (4.207119929516103) + | > loss_mel: 20.706857681274414 (20.553350321777017) + | > loss_duration: 1.3676749467849731 (1.3415754365799066) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.596214294433594 (30.623960885245477) + | > grad_norm_1: tensor(32.6311, device='cuda:0') (tensor(417.0694, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.9951 (0.8754182471643626) + | > loader_time: 0.0159 (0.013159015294535992) + + + --> TIME: 2024-06-25 10:52:49 -- STEP: 416/591 -- GLOBAL_STEP: 29375 + | > loss_disc: 2.6410200595855713 (2.669405300456741) + | > loss_disc_real_0: 0.20635457336902618 (0.2009409598003213) + | > loss_disc_real_1: 0.2212061882019043 (0.22397813104236358) + | > loss_disc_real_2: 0.22699551284313202 (0.23372558697771567) + | > loss_disc_real_3: 0.220040962100029 (0.2340607000514865) + | > loss_disc_real_4: 0.2692941427230835 (0.22733771747264725) + | > loss_disc_real_5: 0.24597874283790588 (0.2384479620780509) + | > loss_0: 2.6410200595855713 (2.669405300456741) + | > grad_norm_0: tensor(8.2442, device='cuda:0') (tensor(12.1358, device='cuda:0')) + | > loss_gen: 1.9599438905715942 (1.9977707086274235) + | > loss_kl: 2.6571643352508545 (2.520159070881511) + | > loss_feat: 4.210910797119141 (4.194428490331536) + | > loss_mel: 20.881864547729492 (20.540191448651818) + | > loss_duration: 1.3656201362609863 (1.3426519789947913) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.075502395629883 (30.595201666538532) + | > grad_norm_1: tensor(294.3127, device='cuda:0') (tensor(411.2569, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 0.9939 (0.8814277029954467) + | > loader_time: 0.0157 (0.013435157445760872) + + + --> TIME: 2024-06-25 10:53:15 -- STEP: 441/591 -- GLOBAL_STEP: 29400 + | > loss_disc: 2.6905875205993652 (2.669387987681795) + | > loss_disc_real_0: 0.20986536145210266 (0.20096810952741273) + | > loss_disc_real_1: 0.24383080005645752 (0.22397409099029578) + | > loss_disc_real_2: 0.2350766807794571 (0.23358379344956404) + | > loss_disc_real_3: 0.21075740456581116 (0.2339767753199385) + | > loss_disc_real_4: 0.22612382471561432 (0.22749551468425327) + | > loss_disc_real_5: 0.25609588623046875 (0.23825914755700126) + | > loss_0: 2.6905875205993652 (2.669387987681795) + | > grad_norm_0: tensor(8.9089, device='cuda:0') (tensor(11.9229, device='cuda:0')) + | > loss_gen: 1.9721996784210205 (1.9951730188058328) + | > loss_kl: 2.6065256595611572 (2.517245823023268) + | > loss_feat: 3.9803943634033203 (4.183212086727284) + | > loss_mel: 20.108564376831055 (20.530362118398784) + | > loss_duration: 1.3532781600952148 (1.3435422498352674) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.02096176147461 (30.569535266244763) + | > grad_norm_1: tensor(428.7360, device='cuda:0') (tensor(400.7610, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 1.0104 (0.8879862815614729) + | > loader_time: 0.0173 (0.013706703575289978) + + + --> TIME: 2024-06-25 10:53:41 -- STEP: 466/591 -- GLOBAL_STEP: 29425 + | > loss_disc: 2.694293975830078 (2.670462715267621) + | > loss_disc_real_0: 0.23286063969135284 (0.20085870298704875) + | > loss_disc_real_1: 0.20252582430839539 (0.22409293276173878) + | > loss_disc_real_2: 0.2391182780265808 (0.23369729115983448) + | > loss_disc_real_3: 0.21648254990577698 (0.23406066070810408) + | > loss_disc_real_4: 0.23368299007415771 (0.22787115291208668) + | > loss_disc_real_5: 0.2552317976951599 (0.23844602470617948) + | > loss_0: 2.694293975830078 (2.670462715267621) + | > grad_norm_0: tensor(5.2860, device='cuda:0') (tensor(11.8940, device='cuda:0')) + | > loss_gen: 1.894287109375 (1.9934734433505648) + | > loss_kl: 2.5120160579681396 (2.5144601528225) + | > loss_feat: 4.063687324523926 (4.172347796321423) + | > loss_mel: 21.352893829345703 (20.51107384513887) + | > loss_duration: 1.3792717456817627 (1.344961148730675) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.20215606689453 (30.536316364108238) + | > grad_norm_1: tensor(61.4417, device='cuda:0') (tensor(391.5032, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 1.0368 (0.8950830406385427) + | > loader_time: 0.0184 (0.013964088177987943) + + + --> TIME: 2024-06-25 10:54:07 -- STEP: 491/591 -- GLOBAL_STEP: 29450 + | > loss_disc: 2.6355669498443604 (2.6697627119037115) + | > loss_disc_real_0: 0.15913918614387512 (0.20063940780104542) + | > loss_disc_real_1: 0.18421721458435059 (0.22401932616340656) + | > loss_disc_real_2: 0.21916061639785767 (0.23360343210687462) + | > loss_disc_real_3: 0.23294289410114288 (0.23409436582426432) + | > loss_disc_real_4: 0.21142083406448364 (0.227492946046433) + | > loss_disc_real_5: 0.2686420679092407 (0.23854809553341563) + | > loss_0: 2.6355669498443604 (2.6697627119037115) + | > grad_norm_0: tensor(14.0180, device='cuda:0') (tensor(11.9508, device='cuda:0')) + | > loss_gen: 2.147406816482544 (1.9941323937320885) + | > loss_kl: 2.4680538177490234 (2.5133503279715086) + | > loss_feat: 3.800132989883423 (4.168729803955477) + | > loss_mel: 20.120058059692383 (20.503993059612817) + | > loss_duration: 1.404405117034912 (1.3462995543256553) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.9400577545166 (30.52650511871773) + | > grad_norm_1: tensor(598.9994, device='cuda:0') (tensor(400.8721, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 1.0803 (0.9022661668462809) + | > loader_time: 0.0166 (0.014214549190886393) + + + --> TIME: 2024-06-25 10:54:35 -- STEP: 516/591 -- GLOBAL_STEP: 29475 + | > loss_disc: 2.668452739715576 (2.6697467417679994) + | > loss_disc_real_0: 0.17990651726722717 (0.20075783656202545) + | > loss_disc_real_1: 0.23961113393306732 (0.22441483297666842) + | > loss_disc_real_2: 0.28264638781547546 (0.23363522431531616) + | > loss_disc_real_3: 0.1650652438402176 (0.23408794622550638) + | > loss_disc_real_4: 0.24191990494728088 (0.22715116266247837) + | > loss_disc_real_5: 0.2441968023777008 (0.23841630447973577) + | > loss_0: 2.668452739715576 (2.6697467417679994) + | > grad_norm_0: tensor(13.2373, device='cuda:0') (tensor(12.0731, device='cuda:0')) + | > loss_gen: 1.9562137126922607 (1.9955753499223263) + | > loss_kl: 2.5187647342681885 (2.5119678586028322) + | > loss_feat: 4.195611953735352 (4.16482408702835) + | > loss_mel: 20.472858428955078 (20.497286415839373) + | > loss_duration: 1.3741381168365479 (1.348002133443374) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.51758575439453 (30.51765580140343) + | > grad_norm_1: tensor(578.5282, device='cuda:0') (tensor(410.8280, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 1.079 (0.9109071031097291) + | > loader_time: 0.0252 (0.014490019905474757) + + + --> TIME: 2024-06-25 10:55:04 -- STEP: 541/591 -- GLOBAL_STEP: 29500 + | > loss_disc: 2.680213212966919 (2.669326998610151) + | > loss_disc_real_0: 0.19580420851707458 (0.2006464425660326) + | > loss_disc_real_1: 0.2983524203300476 (0.2243697002061855) + | > loss_disc_real_2: 0.24865837395191193 (0.23368763945680007) + | > loss_disc_real_3: 0.20292827486991882 (0.2343024808961671) + | > loss_disc_real_4: 0.24531465768814087 (0.22690948304098768) + | > loss_disc_real_5: 0.26138919591903687 (0.23844051220610932) + | > loss_0: 2.680213212966919 (2.669326998610151) + | > grad_norm_0: tensor(9.8379, device='cuda:0') (tensor(12.1536, device='cuda:0')) + | > loss_gen: 1.926344394683838 (1.9959006836145494) + | > loss_kl: 2.43269944190979 (2.5093452207702795) + | > loss_feat: 3.8665974140167236 (4.158098193942504) + | > loss_mel: 19.480342864990234 (20.482775778074135) + | > loss_duration: 1.3673487901687622 (1.349619125262206) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.073333740234375 (30.495738962000708) + | > grad_norm_1: tensor(818.6430, device='cuda:0') (tensor(423.5204, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 1.137 (0.9210128563829797) + | > loader_time: 0.0195 (0.014777180007116631) + + + --> TIME: 2024-06-25 10:55:34 -- STEP: 566/591 -- GLOBAL_STEP: 29525 + | > loss_disc: 2.7425696849823 (2.6704326514220473) + | > loss_disc_real_0: 0.2406763732433319 (0.20075044184659904) + | > loss_disc_real_1: 0.22841590642929077 (0.22437834460617384) + | > loss_disc_real_2: 0.2343626618385315 (0.23370865455996442) + | > loss_disc_real_3: 0.23968742787837982 (0.23425836995401988) + | > loss_disc_real_4: 0.24524769186973572 (0.2270652344549081) + | > loss_disc_real_5: 0.2624388039112091 (0.2385164590901284) + | > loss_0: 2.7425696849823 (2.6704326514220473) + | > grad_norm_0: tensor(11.1439, device='cuda:0') (tensor(12.2181, device='cuda:0')) + | > loss_gen: 1.990382194519043 (1.994109303706946) + | > loss_kl: 2.403275728225708 (2.5066363099607045) + | > loss_feat: 3.704045295715332 (4.146403947061864) + | > loss_mel: 19.75767707824707 (20.466709386333555) + | > loss_duration: 1.3951903581619263 (1.3513162675257706) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.250572204589844 (30.465175177099006) + | > grad_norm_1: tensor(394.0545, device='cuda:0') (tensor(424.2818, device='cuda:0')) + | > current_lr_0: 0.00019877866781345852 + | > current_lr_1: 0.00019877866781345852 + | > step_time: 1.1903 (0.9320878304356819) + | > loader_time: 0.0193 (0.015104668300480387) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035234664348845787 (-8.727134542262311e-05) + | > avg_loss_disc: 2.6486997655097473 (-0.0656152836819901) + | > avg_loss_disc_real_0: 0.20657845396310726 (-0.05606209867178119) + | > avg_loss_disc_real_1: 0.2295118586180058 (+0.03649642905022235) + | > avg_loss_disc_real_2: 0.2374801515264714 (+0.015607363049020162) + | > avg_loss_disc_real_3: 0.23889416646450123 (+0.010550805545867764) + | > avg_loss_disc_real_4: 0.20586393995487945 (-0.029997318666032002) + | > avg_loss_disc_real_5: 0.2126206045455121 (-0.016546464980916775) + | > avg_loss_0: 2.6486997655097473 (-0.0656152836819901) + | > avg_loss_gen: 1.9220048594982067 (+0.03335164709294092) + | > avg_loss_kl: 2.589354652039549 (+0.05351024992922637) + | > avg_loss_feat: 4.096872593494172 (+0.24905454858820564) + | > avg_loss_mel: 20.345974455488495 (-0.1962604116886162) + | > avg_loss_duration: 1.363717365772166 (+0.002928137779235618) + | > avg_loss_1: 30.317923769037773 (+0.14258400937344007) + + + > EPOCH: 51/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 10:56:12)  + + --> TIME: 2024-06-25 10:56:14 -- STEP: 0/591 -- GLOBAL_STEP: 29550 + | > loss_disc: 2.662083625793457 (2.662083625793457) + | > loss_disc_real_0: 0.19647055864334106 (0.19647055864334106) + | > loss_disc_real_1: 0.21134616434574127 (0.21134616434574127) + | > loss_disc_real_2: 0.22269441187381744 (0.22269441187381744) + | > loss_disc_real_3: 0.22073492407798767 (0.22073492407798767) + | > loss_disc_real_4: 0.19539955258369446 (0.19539955258369446) + | > loss_disc_real_5: 0.20228096842765808 (0.20228096842765808) + | > loss_0: 2.662083625793457 (2.662083625793457) + | > grad_norm_0: tensor(9.7947, device='cuda:0') (tensor(9.7947, device='cuda:0')) + | > loss_gen: 2.2198877334594727 (2.2198877334594727) + | > loss_kl: 2.6702194213867188 (2.6702194213867188) + | > loss_feat: 4.464156150817871 (4.464156150817871) + | > loss_mel: 20.521148681640625 (20.521148681640625) + | > loss_duration: 1.3162105083465576 (1.3162105083465576) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.191621780395508 (31.191621780395508) + | > grad_norm_1: tensor(747.3526, device='cuda:0') (tensor(747.3526, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.8549 (0.8549227714538574) + | > loader_time: 0.6401 (0.6400604248046875) + + + --> TIME: 2024-06-25 10:56:33 -- STEP: 25/591 -- GLOBAL_STEP: 29575 + | > loss_disc: 2.7326135635375977 (2.691865396499633) + | > loss_disc_real_0: 0.2618033289909363 (0.21312849938869477) + | > loss_disc_real_1: 0.20737631618976593 (0.2216743391752243) + | > loss_disc_real_2: 0.22914941608905792 (0.23356543064117433) + | > loss_disc_real_3: 0.24430477619171143 (0.2383961969614029) + | > loss_disc_real_4: 0.18720509111881256 (0.22521110534667968) + | > loss_disc_real_5: 0.2280065268278122 (0.23944544553756714) + | > loss_0: 2.7326135635375977 (2.691865396499633) + | > grad_norm_0: tensor(13.3020, device='cuda:0') (tensor(13.6340, device='cuda:0')) + | > loss_gen: 1.893142580986023 (2.01570716381073) + | > loss_kl: 2.571401596069336 (2.6122701263427732) + | > loss_feat: 4.0987324714660645 (4.321314830780029) + | > loss_mel: 20.65045928955078 (20.593282623291017) + | > loss_duration: 1.2729281187057495 (1.3153028726577756) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.486663818359375 (30.857877349853517) + | > grad_norm_1: tensor(260.9255, device='cuda:0') (tensor(576.6981, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.7916 (0.7775754070281983) + | > loader_time: 0.0091 (0.008776063919067382) + + + --> TIME: 2024-06-25 10:56:54 -- STEP: 50/591 -- GLOBAL_STEP: 29600 + | > loss_disc: 2.588524103164673 (2.6761519479751588) + | > loss_disc_real_0: 0.16391533613204956 (0.20478603959083558) + | > loss_disc_real_1: 0.19616463780403137 (0.22554820537567138) + | > loss_disc_real_2: 0.2092282921075821 (0.23188685178756713) + | > loss_disc_real_3: 0.1967867910861969 (0.23553153455257417) + | > loss_disc_real_4: 0.19976851344108582 (0.2233300644159317) + | > loss_disc_real_5: 0.20993594825267792 (0.23808385103940963) + | > loss_0: 2.588524103164673 (2.6761519479751588) + | > grad_norm_0: tensor(13.9686, device='cuda:0') (tensor(13.3001, device='cuda:0')) + | > loss_gen: 2.4540488719940186 (2.019773595333099) + | > loss_kl: 2.513244390487671 (2.5806163311004635) + | > loss_feat: 4.884783744812012 (4.301843090057372) + | > loss_mel: 21.40984344482422 (20.660450134277344) + | > loss_duration: 1.3672388792037964 (1.3283009958267211) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.62915802001953 (30.89098400115967) + | > grad_norm_1: tensor(1192.5011, device='cuda:0') (tensor(603.1380, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.8075 (0.7909200477600098) + | > loader_time: 0.0099 (0.009363698959350587) + + + --> TIME: 2024-06-25 10:57:15 -- STEP: 75/591 -- GLOBAL_STEP: 29625 + | > loss_disc: 2.6459498405456543 (2.6735351562500003) + | > loss_disc_real_0: 0.1621990203857422 (0.2044313548008601) + | > loss_disc_real_1: 0.2357778400182724 (0.22513007978598276) + | > loss_disc_real_2: 0.20127278566360474 (0.2324539029598236) + | > loss_disc_real_3: 0.18174298107624054 (0.23498343368371327) + | > loss_disc_real_4: 0.19884353876113892 (0.22425426185131073) + | > loss_disc_real_5: 0.23157794773578644 (0.23847366432348888) + | > loss_0: 2.6459498405456543 (2.6735351562500003) + | > grad_norm_0: tensor(11.6352, device='cuda:0') (tensor(14.1315, device='cuda:0')) + | > loss_gen: 2.178351402282715 (2.012604935963949) + | > loss_kl: 2.638597249984741 (2.5712810707092277) + | > loss_feat: 4.196428298950195 (4.27953873316447) + | > loss_mel: 20.836214065551758 (20.68008260091146) + | > loss_duration: 1.3508517742156982 (1.3329244168599448) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.200441360473633 (30.876431528727213) + | > grad_norm_1: tensor(48.7492, device='cuda:0') (tensor(579.6764, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.8178 (0.7994395224253339) + | > loader_time: 0.0102 (0.009771496454874674) + + + --> TIME: 2024-06-25 10:57:36 -- STEP: 100/591 -- GLOBAL_STEP: 29650 + | > loss_disc: 2.6030712127685547 (2.664256904125213) + | > loss_disc_real_0: 0.19501681625843048 (0.20323856964707374) + | > loss_disc_real_1: 0.23763245344161987 (0.2242246748507023) + | > loss_disc_real_2: 0.25353652238845825 (0.23313817769289016) + | > loss_disc_real_3: 0.24916112422943115 (0.23476878583431243) + | > loss_disc_real_4: 0.2148812860250473 (0.2225075951218605) + | > loss_disc_real_5: 0.24640905857086182 (0.23884770900011063) + | > loss_0: 2.6030712127685547 (2.664256904125213) + | > grad_norm_0: tensor(10.1154, device='cuda:0') (tensor(14.1483, device='cuda:0')) + | > loss_gen: 2.027101516723633 (2.0176529633998874) + | > loss_kl: 2.591874361038208 (2.5613016605377186) + | > loss_feat: 4.012483596801758 (4.295990698337555) + | > loss_mel: 19.89932632446289 (20.692321052551268) + | > loss_duration: 1.3195583820343018 (1.3394183170795453) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.850345611572266 (30.906684532165528) + | > grad_norm_1: tensor(588.9790, device='cuda:0') (tensor(628.5945, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.8412 (0.8065994215011598) + | > loader_time: 0.0105 (0.01003254652023315) + + + --> TIME: 2024-06-25 10:57:57 -- STEP: 125/591 -- GLOBAL_STEP: 29675 + | > loss_disc: 2.6500463485717773 (2.6627628879547114) + | > loss_disc_real_0: 0.327764093875885 (0.20335109663009643) + | > loss_disc_real_1: 0.1820375770330429 (0.22379554724693299) + | > loss_disc_real_2: 0.23713815212249756 (0.2328435057401657) + | > loss_disc_real_3: 0.20778784155845642 (0.23449249124526977) + | > loss_disc_real_4: 0.21480169892311096 (0.2226180419921875) + | > loss_disc_real_5: 0.25382229685783386 (0.23859980070590972) + | > loss_0: 2.6500463485717773 (2.6627628879547114) + | > grad_norm_0: tensor(21.4046, device='cuda:0') (tensor(13.9490, device='cuda:0')) + | > loss_gen: 1.842799186706543 (2.0201643066406256) + | > loss_kl: 2.5012106895446777 (2.550988739013671) + | > loss_feat: 4.490828990936279 (4.297339548110959) + | > loss_mel: 21.507312774658203 (20.727915740966797) + | > loss_duration: 1.38836669921875 (1.340653805732728) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.730518341064453 (30.937061935424804) + | > grad_norm_1: tensor(485.1628, device='cuda:0') (tensor(636.9637, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.8382 (0.8128598423004152) + | > loader_time: 0.0109 (0.010226509094238277) + + + --> TIME: 2024-06-25 10:58:19 -- STEP: 150/591 -- GLOBAL_STEP: 29700 + | > loss_disc: 2.677219867706299 (2.6644695583979283) + | > loss_disc_real_0: 0.1378602683544159 (0.2023555897176266) + | > loss_disc_real_1: 0.22314850986003876 (0.2241957414150238) + | > loss_disc_real_2: 0.23123681545257568 (0.2330075779557228) + | > loss_disc_real_3: 0.22442689538002014 (0.23448969632387162) + | > loss_disc_real_4: 0.26668697595596313 (0.22434281210104626) + | > loss_disc_real_5: 0.24390794336795807 (0.23849144846200943) + | > loss_0: 2.677219867706299 (2.6644695583979283) + | > grad_norm_0: tensor(10.7220, device='cuda:0') (tensor(13.5338, device='cuda:0')) + | > loss_gen: 1.9478963613510132 (2.0135710946718848) + | > loss_kl: 2.60689640045166 (2.5421770445505776) + | > loss_feat: 4.264865875244141 (4.280984864234924) + | > loss_mel: 20.67646598815918 (20.684906946818035) + | > loss_duration: 1.284947156906128 (1.3392238235473641) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.78107261657715 (30.860863596598307) + | > grad_norm_1: tensor(253.8314, device='cuda:0') (tensor(574.8434, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.876 (0.8196339193979899) + | > loader_time: 0.0117 (0.01045393308003743) + + + --> TIME: 2024-06-25 10:58:41 -- STEP: 175/591 -- GLOBAL_STEP: 29725 + | > loss_disc: 2.67079758644104 (2.671589509419034) + | > loss_disc_real_0: 0.08841857314109802 (0.20277115625994555) + | > loss_disc_real_1: 0.22918866574764252 (0.22458869465759823) + | > loss_disc_real_2: 0.23382149636745453 (0.23317210716860634) + | > loss_disc_real_3: 0.21524469554424286 (0.234835929444858) + | > loss_disc_real_4: 0.2298378199338913 (0.2256778349195208) + | > loss_disc_real_5: 0.23280277848243713 (0.23909839017050608) + | > loss_0: 2.67079758644104 (2.671589509419034) + | > grad_norm_0: tensor(25.3242, device='cuda:0') (tensor(13.5507, device='cuda:0')) + | > loss_gen: 2.0398097038269043 (2.005514344487871) + | > loss_kl: 2.5166962146759033 (2.53121695790972) + | > loss_feat: 3.904533624649048 (4.251153376443047) + | > loss_mel: 20.666934967041016 (20.65514421735492) + | > loss_duration: 1.322371244430542 (1.3369918673379093) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.450345993041992 (30.78002061026437) + | > grad_norm_1: tensor(102.9142, device='cuda:0') (tensor(528.1959, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.8609 (0.826709773199899) + | > loader_time: 0.0116 (0.010689593723842077) + + + --> TIME: 2024-06-25 10:59:04 -- STEP: 200/591 -- GLOBAL_STEP: 29750 + | > loss_disc: 2.64412522315979 (2.66996166110039) + | > loss_disc_real_0: 0.1631864309310913 (0.20242981456220158) + | > loss_disc_real_1: 0.2095702737569809 (0.2240037954598665) + | > loss_disc_real_2: 0.2259434014558792 (0.23282429203391075) + | > loss_disc_real_3: 0.17980098724365234 (0.23452423006296158) + | > loss_disc_real_4: 0.20990055799484253 (0.22650721788406372) + | > loss_disc_real_5: 0.23293903470039368 (0.23878708742558957) + | > loss_0: 2.64412522315979 (2.66996166110039) + | > grad_norm_0: tensor(6.3992, device='cuda:0') (tensor(12.8064, device='cuda:0')) + | > loss_gen: 1.993632435798645 (2.000270040631293) + | > loss_kl: 2.5261213779449463 (2.5252184832096116) + | > loss_feat: 4.197257995605469 (4.2433961176872295) + | > loss_mel: 20.419527053833008 (20.643265390396117) + | > loss_duration: 1.3246922492980957 (1.3367249423265468) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.461231231689453 (30.748874826431276) + | > grad_norm_1: tensor(70.1924, device='cuda:0') (tensor(481.4918, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.9025 (0.8345475172996523) + | > loader_time: 0.0145 (0.01092584490776062) + + + --> TIME: 2024-06-25 10:59:27 -- STEP: 225/591 -- GLOBAL_STEP: 29775 + | > loss_disc: 2.685636520385742 (2.671339249081084) + | > loss_disc_real_0: 0.2151235193014145 (0.20241703788439439) + | > loss_disc_real_1: 0.2384866327047348 (0.22415458420912424) + | > loss_disc_real_2: 0.25852394104003906 (0.2330616186062495) + | > loss_disc_real_3: 0.24503737688064575 (0.23450941688484617) + | > loss_disc_real_4: 0.2373160868883133 (0.22707134571340348) + | > loss_disc_real_5: 0.22344116866588593 (0.23871407045258417) + | > loss_0: 2.685636520385742 (2.671339249081084) + | > grad_norm_0: tensor(9.6580, device='cuda:0') (tensor(12.4816, device='cuda:0')) + | > loss_gen: 1.9503557682037354 (1.996795279184977) + | > loss_kl: 2.561589479446411 (2.5218233182695187) + | > loss_feat: 4.120251178741455 (4.229456225501169) + | > loss_mel: 20.839305877685547 (20.643509572347003) + | > loss_duration: 1.3367948532104492 (1.3360187721252452) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.80829620361328 (30.72760299682617) + | > grad_norm_1: tensor(490.8036, device='cuda:0') (tensor(446.9346, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.8944 (0.8414053408304851) + | > loader_time: 0.0127 (0.01121893776787652) + + + --> TIME: 2024-06-25 10:59:49 -- STEP: 250/591 -- GLOBAL_STEP: 29800 + | > loss_disc: 2.6605045795440674 (2.673733292579653) + | > loss_disc_real_0: 0.20883844792842865 (0.20241435676813133) + | > loss_disc_real_1: 0.2139521688222885 (0.22453941416740417) + | > loss_disc_real_2: 0.20826837420463562 (0.2332012481689453) + | > loss_disc_real_3: 0.20620353519916534 (0.23444093680381775) + | > loss_disc_real_4: 0.2280399203300476 (0.22777277034521104) + | > loss_disc_real_5: 0.2552456557750702 (0.23871719366312028) + | > loss_0: 2.6605045795440674 (2.673733292579653) + | > grad_norm_0: tensor(8.8181, device='cuda:0') (tensor(12.2392, device='cuda:0')) + | > loss_gen: 1.9516029357910156 (1.9930140066146849) + | > loss_kl: 2.459017753601074 (2.5177157258987437) + | > loss_feat: 4.161557197570801 (4.220678130149842) + | > loss_mel: 20.05814552307129 (20.625467704772948) + | > loss_duration: 1.3115382194519043 (1.3367940807342538) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.941862106323242 (30.69366947937012) + | > grad_norm_1: tensor(136.1990, device='cuda:0') (tensor(419.2625, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.8852 (0.8468123455047608) + | > loader_time: 0.0133 (0.011501261711120606) + + + --> TIME: 2024-06-25 11:00:12 -- STEP: 275/591 -- GLOBAL_STEP: 29825 + | > loss_disc: 2.6547460556030273 (2.6759032812985524) + | > loss_disc_real_0: 0.2771238088607788 (0.2027882645346902) + | > loss_disc_real_1: 0.23807917535305023 (0.2243470301953229) + | > loss_disc_real_2: 0.24736462533473969 (0.23375166963447225) + | > loss_disc_real_3: 0.24354666471481323 (0.23453296011144464) + | > loss_disc_real_4: 0.24124030768871307 (0.22852313621477643) + | > loss_disc_real_5: 0.24143880605697632 (0.23876283694397318) + | > loss_0: 2.6547460556030273 (2.6759032812985524) + | > grad_norm_0: tensor(14.0371, device='cuda:0') (tensor(12.0683, device='cuda:0')) + | > loss_gen: 1.8332812786102295 (1.988596949143843) + | > loss_kl: 2.3929696083068848 (2.5159379369562327) + | > loss_feat: 4.121125221252441 (4.206044310656464) + | > loss_mel: 19.952001571655273 (20.607646643898704) + | > loss_duration: 1.348593831062317 (1.3367133929512727) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.647972106933594 (30.65493907581676) + | > grad_norm_1: tensor(277.5544, device='cuda:0') (tensor(396.3137, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.9147 (0.8518941272388806) + | > loader_time: 0.016 (0.011794081601229581) + + + --> TIME: 2024-06-25 11:00:36 -- STEP: 300/591 -- GLOBAL_STEP: 29850 + | > loss_disc: 2.662017822265625 (2.6753792381286634) + | > loss_disc_real_0: 0.17327053844928741 (0.20235551449159786) + | > loss_disc_real_1: 0.18155533075332642 (0.22425609489281972) + | > loss_disc_real_2: 0.23972594738006592 (0.23364436691006024) + | > loss_disc_real_3: 0.2510121464729309 (0.23441706721981367) + | > loss_disc_real_4: 0.2163195013999939 (0.2287277634938557) + | > loss_disc_real_5: 0.2522807717323303 (0.2386498810350895) + | > loss_0: 2.662017822265625 (2.6753792381286634) + | > grad_norm_0: tensor(7.7843, device='cuda:0') (tensor(12.1638, device='cuda:0')) + | > loss_gen: 1.9263603687286377 (1.9880504767100016) + | > loss_kl: 2.489748954772949 (2.5144202303886414) + | > loss_feat: 4.1591410636901855 (4.203767036596935) + | > loss_mel: 20.215023040771484 (20.585198923746745) + | > loss_duration: 1.3571643829345703 (1.3370005742708848) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.147438049316406 (30.628437074025474) + | > grad_norm_1: tensor(527.6698, device='cuda:0') (tensor(384.8951, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.9156 (0.8567656747500102) + | > loader_time: 0.0134 (0.012043588956197103) + + + --> TIME: 2024-06-25 11:00:59 -- STEP: 325/591 -- GLOBAL_STEP: 29875 + | > loss_disc: 2.5535366535186768 (2.6729978781480055) + | > loss_disc_real_0: 0.1573689579963684 (0.20177253752946864) + | > loss_disc_real_1: 0.16082477569580078 (0.22425384548994212) + | > loss_disc_real_2: 0.20013278722763062 (0.23321924214179698) + | > loss_disc_real_3: 0.22628723084926605 (0.23415568906527298) + | > loss_disc_real_4: 0.22153881192207336 (0.22875165434984052) + | > loss_disc_real_5: 0.23142264783382416 (0.23857728311648735) + | > loss_0: 2.5535366535186768 (2.6729978781480055) + | > grad_norm_0: tensor(7.8205, device='cuda:0') (tensor(11.9702, device='cuda:0')) + | > loss_gen: 2.0865190029144287 (1.9889556811406062) + | > loss_kl: 2.4123339653015137 (2.5118894503666795) + | > loss_feat: 4.486124515533447 (4.2036906880598846) + | > loss_mel: 21.307432174682617 (20.587019688532905) + | > loss_duration: 1.3702192306518555 (1.3377132100325368) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.662628173828125 (30.62926856994629) + | > grad_norm_1: tensor(343.5533, device='cuda:0') (tensor(383.7762, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.9337 (0.8617823637448825) + | > loader_time: 0.0168 (0.012319112190833453) + + + --> TIME: 2024-06-25 11:01:23 -- STEP: 350/591 -- GLOBAL_STEP: 29900 + | > loss_disc: 2.693176507949829 (2.6720924220766338) + | > loss_disc_real_0: 0.13742946088314056 (0.2013327735875335) + | > loss_disc_real_1: 0.20940226316452026 (0.22418636500835418) + | > loss_disc_real_2: 0.20795968174934387 (0.23343110190970556) + | > loss_disc_real_3: 0.2864905893802643 (0.23428223554577146) + | > loss_disc_real_4: 0.24028119444847107 (0.22847055469240454) + | > loss_disc_real_5: 0.2478751540184021 (0.23852854818105698) + | > loss_0: 2.693176507949829 (2.6720924220766338) + | > grad_norm_0: tensor(24.6381, device='cuda:0') (tensor(11.9708, device='cuda:0')) + | > loss_gen: 2.293123245239258 (1.990711498260498) + | > loss_kl: 2.4398696422576904 (2.5102632638386306) + | > loss_feat: 4.328922748565674 (4.203305617741182) + | > loss_mel: 20.771015167236328 (20.582470610482364) + | > loss_duration: 1.3693736791610718 (1.3391971482549394) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.202302932739258 (30.625948039463587) + | > grad_norm_1: tensor(662.2964, device='cuda:0') (tensor(391.3366, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.9413 (0.8673341764722552) + | > loader_time: 0.0153 (0.012594578606741763) + + + --> TIME: 2024-06-25 11:01:48 -- STEP: 375/591 -- GLOBAL_STEP: 29925 + | > loss_disc: 2.6968092918395996 (2.6725353775024403) + | > loss_disc_real_0: 0.16626021265983582 (0.2017012309829395) + | > loss_disc_real_1: 0.24978859722614288 (0.22414031692345937) + | > loss_disc_real_2: 0.21387991309165955 (0.2334795176188151) + | > loss_disc_real_3: 0.21902219951152802 (0.23423979556560517) + | > loss_disc_real_4: 0.23091433942317963 (0.22842704415321344) + | > loss_disc_real_5: 0.23924386501312256 (0.23859896528720856) + | > loss_0: 2.6968092918395996 (2.6725353775024403) + | > grad_norm_0: tensor(11.5499, device='cuda:0') (tensor(12.2074, device='cuda:0')) + | > loss_gen: 1.904251217842102 (1.9901490580240886) + | > loss_kl: 2.5178110599517822 (2.5075833880106617) + | > loss_feat: 4.135335445404053 (4.194519079844162) + | > loss_mel: 20.524255752563477 (20.56969972229006) + | > loss_duration: 1.3554702997207642 (1.3405913661321) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.437124252319336 (30.602542516072592) + | > grad_norm_1: tensor(362.0538, device='cuda:0') (tensor(402.5597, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.9679 (0.8727376658121745) + | > loader_time: 0.0171 (0.012914731343587236) + + + --> TIME: 2024-06-25 11:02:13 -- STEP: 400/591 -- GLOBAL_STEP: 29950 + | > loss_disc: 2.621248245239258 (2.6727039504051198) + | > loss_disc_real_0: 0.22838068008422852 (0.20150679266080268) + | > loss_disc_real_1: 0.21899542212486267 (0.2242843806371093) + | > loss_disc_real_2: 0.2524661123752594 (0.23358840543776752) + | > loss_disc_real_3: 0.2454816699028015 (0.23424044277518988) + | > loss_disc_real_4: 0.22563444077968597 (0.22849792420864098) + | > loss_disc_real_5: 0.2414795607328415 (0.23848876871168614) + | > loss_0: 2.621248245239258 (2.6727039504051198) + | > grad_norm_0: tensor(12.3574, device='cuda:0') (tensor(12.1694, device='cuda:0')) + | > loss_gen: 2.0540876388549805 (1.9890797185897826) + | > loss_kl: 2.5000884532928467 (2.5061857694387446) + | > loss_feat: 4.204702854156494 (4.185955656170849) + | > loss_mel: 20.521564483642578 (20.55284606933596) + | > loss_duration: 1.3689531087875366 (1.3416212886571879) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.649398803710938 (30.57568841934204) + | > grad_norm_1: tensor(770.6968, device='cuda:0') (tensor(401.1243, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.9592 (0.8792703747749329) + | > loader_time: 0.0171 (0.013214065432548518) + + + --> TIME: 2024-06-25 11:02:38 -- STEP: 425/591 -- GLOBAL_STEP: 29975 + | > loss_disc: 2.658022880554199 (2.671029220468857) + | > loss_disc_real_0: 0.2514830231666565 (0.20108503234737066) + | > loss_disc_real_1: 0.22827287018299103 (0.22441012424581191) + | > loss_disc_real_2: 0.2587577700614929 (0.23371077681289) + | > loss_disc_real_3: 0.2753790020942688 (0.23419242164667914) + | > loss_disc_real_4: 0.22439081966876984 (0.22791217958225918) + | > loss_disc_real_5: 0.2448960840702057 (0.23848613777581384) + | > loss_0: 2.658022880554199 (2.671029220468857) + | > grad_norm_0: tensor(19.6327, device='cuda:0') (tensor(12.2680, device='cuda:0')) + | > loss_gen: 1.8294881582260132 (1.9909701970044305) + | > loss_kl: 2.51279878616333 (2.504640101264506) + | > loss_feat: 3.932239055633545 (4.1824332422368675) + | > loss_mel: 19.82972526550293 (20.539708525713774) + | > loss_duration: 1.3719655275344849 (1.342459849750294) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.47621726989746 (30.560211841358857) + | > grad_norm_1: tensor(669.7407, device='cuda:0') (tensor(417.2208, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 1.0134 (0.8861039301928352) + | > loader_time: 0.019 (0.013481901954202086) + + + --> TIME: 2024-06-25 11:03:04 -- STEP: 450/591 -- GLOBAL_STEP: 30000 + | > loss_disc: 2.6965484619140625 (2.6723338021172407) + | > loss_disc_real_0: 0.20284996926784515 (0.20132050870193383) + | > loss_disc_real_1: 0.1973075568675995 (0.22431251840458977) + | > loss_disc_real_2: 0.29497694969177246 (0.23401575562026766) + | > loss_disc_real_3: 0.30392536520957947 (0.23440872463915083) + | > loss_disc_real_4: 0.25534477829933167 (0.228165735900402) + | > loss_disc_real_5: 0.27158015966415405 (0.23845243215560913) + | > loss_0: 2.6965484619140625 (2.6723338021172407) + | > grad_norm_0: tensor(7.8755, device='cuda:0') (tensor(12.3955, device='cuda:0')) + | > loss_gen: 1.8938888311386108 (1.9906942121187845) + | > loss_kl: 2.545610189437866 (2.504423950513205) + | > loss_feat: 3.8264949321746826 (4.173809955914814) + | > loss_mel: 20.006210327148438 (20.52871134016251) + | > loss_duration: 1.3733654022216797 (1.343657144705454) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.64556884765625 (30.54129653506809) + | > grad_norm_1: tensor(566.7639, device='cuda:0') (tensor(419.8984, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 0.9906 (0.8933548752466838) + | > loader_time: 0.0225 (0.013779631190829802) + + + > CHECKPOINT : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/checkpoint_30000.pth + + --> TIME: 2024-06-25 11:03:34 -- STEP: 475/591 -- GLOBAL_STEP: 30025 + | > loss_disc: 2.7291603088378906 (2.672346386658517) + | > loss_disc_real_0: 0.26467081904411316 (0.20116803722946272) + | > loss_disc_real_1: 0.22537317872047424 (0.22436006367206573) + | > loss_disc_real_2: 0.23857110738754272 (0.23385300231607337) + | > loss_disc_real_3: 0.24814510345458984 (0.23440765779269368) + | > loss_disc_real_4: 0.2432243973016739 (0.22796941801121354) + | > loss_disc_real_5: 0.2531489431858063 (0.23850273276630202) + | > loss_0: 2.7291603088378906 (2.672346386658517) + | > grad_norm_0: tensor(16.7258, device='cuda:0') (tensor(12.4237, device='cuda:0')) + | > loss_gen: 1.8945446014404297 (1.9897426010432997) + | > loss_kl: 2.5084714889526367 (2.5020553829795444) + | > loss_feat: 3.8543198108673096 (4.165222560481021) + | > loss_mel: 20.595888137817383 (20.50915652626441) + | > loss_duration: 1.3889343738555908 (1.3448609655781791) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.242158889770508 (30.511037971095035) + | > grad_norm_1: tensor(728.0768, device='cuda:0') (tensor(426.4171, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 1.0287 (0.9010886041741623) + | > loader_time: 0.0163 (0.014078393735383683) + + + --> TIME: 2024-06-25 11:04:01 -- STEP: 500/591 -- GLOBAL_STEP: 30050 + | > loss_disc: 2.6460537910461426 (2.6723950967788688) + | > loss_disc_real_0: 0.20408512651920319 (0.2010416430085898) + | > loss_disc_real_1: 0.18045957386493683 (0.22429635980725288) + | > loss_disc_real_2: 0.23734194040298462 (0.2337983896136284) + | > loss_disc_real_3: 0.24400939047336578 (0.23444629734754563) + | > loss_disc_real_4: 0.20387588441371918 (0.22798450055718417) + | > loss_disc_real_5: 0.22894644737243652 (0.2385667079091072) + | > loss_0: 2.6460537910461426 (2.6723950967788688) + | > grad_norm_0: tensor(7.6879, device='cuda:0') (tensor(12.3897, device='cuda:0')) + | > loss_gen: 1.8925758600234985 (1.9893986628055573) + | > loss_kl: 2.401653289794922 (2.499687774658204) + | > loss_feat: 4.218636989593506 (4.160494857788086) + | > loss_mel: 20.284576416015625 (20.490733806610127) + | > loss_duration: 1.3870145082473755 (1.3462926709651943) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.184457778930664 (30.486607719421386) + | > grad_norm_1: tensor(521.3198, device='cuda:0') (tensor(429.2189, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 1.0751 (0.9098460493087769) + | > loader_time: 0.025 (0.014412104129791256) + + + --> TIME: 2024-06-25 11:04:29 -- STEP: 525/591 -- GLOBAL_STEP: 30075 + | > loss_disc: 2.6985602378845215 (2.6729449281238367) + | > loss_disc_real_0: 0.27044105529785156 (0.20139130443334585) + | > loss_disc_real_1: 0.20654603838920593 (0.22442067245642341) + | > loss_disc_real_2: 0.24223220348358154 (0.23383009237902505) + | > loss_disc_real_3: 0.22806470096111298 (0.23434312008676075) + | > loss_disc_real_4: 0.21057438850402832 (0.22793219753674093) + | > loss_disc_real_5: 0.24604575335979462 (0.2386042921316056) + | > loss_0: 2.6985602378845215 (2.6729449281238367) + | > grad_norm_0: tensor(11.2138, device='cuda:0') (tensor(12.5302, device='cuda:0')) + | > loss_gen: 1.7374210357666016 (1.9892131616955713) + | > loss_kl: 2.464970350265503 (2.497821921848116) + | > loss_feat: 3.9471561908721924 (4.1525132292792915) + | > loss_mel: 20.1290340423584 (20.475754746936634) + | > loss_duration: 1.3959678411483765 (1.3482742041633238) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.674549102783203 (30.463577197846913) + | > grad_norm_1: tensor(264.5758, device='cuda:0') (tensor(434.6346, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 1.0669 (0.9187668482462565) + | > loader_time: 0.0224 (0.01471615700494675) + + + --> TIME: 2024-06-25 11:04:59 -- STEP: 550/591 -- GLOBAL_STEP: 30100 + | > loss_disc: 2.700697660446167 (2.672806346199728) + | > loss_disc_real_0: 0.24915236234664917 (0.2013469629802488) + | > loss_disc_real_1: 0.3002597689628601 (0.2245546592094681) + | > loss_disc_real_2: 0.24853168427944183 (0.2338328851894899) + | > loss_disc_real_3: 0.2218729704618454 (0.2343644870140336) + | > loss_disc_real_4: 0.20689897239208221 (0.22775685938921839) + | > loss_disc_real_5: 0.22276020050048828 (0.23851698515090075) + | > loss_0: 2.700697660446167 (2.672806346199728) + | > grad_norm_0: tensor(16.9959, device='cuda:0') (tensor(12.5856, device='cuda:0')) + | > loss_gen: 1.960153341293335 (1.990527185310017) + | > loss_kl: 2.497662305831909 (2.4950723773782912) + | > loss_feat: 4.012714385986328 (4.148267644535411) + | > loss_mel: 20.45844078063965 (20.461691821705223) + | > loss_duration: 1.3701987266540527 (1.3496749949455258) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.299171447753906 (30.44523396925493) + | > grad_norm_1: tensor(738.9987, device='cuda:0') (tensor(443.0742, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 1.1394 (0.9292546293952246) + | > loader_time: 0.0262 (0.015068608197298914) + + + --> TIME: 2024-06-25 11:05:30 -- STEP: 575/591 -- GLOBAL_STEP: 30125 + | > loss_disc: 2.638380289077759 (2.672666448510209) + | > loss_disc_real_0: 0.18035683035850525 (0.20124528764382654) + | > loss_disc_real_1: 0.28453516960144043 (0.2245696339918219) + | > loss_disc_real_2: 0.24654819071292877 (0.23372796022373696) + | > loss_disc_real_3: 0.24006135761737823 (0.2344099174115969) + | > loss_disc_real_4: 0.21055929362773895 (0.22765257783558057) + | > loss_disc_real_5: 0.2370760291814804 (0.23850097700305606) + | > loss_0: 2.638380289077759 (2.672666448510209) + | > grad_norm_0: tensor(14.5254, device='cuda:0') (tensor(12.6197, device='cuda:0')) + | > loss_gen: 2.1392903327941895 (1.9903522503894309) + | > loss_kl: 2.4044322967529297 (2.4922752720376717) + | > loss_feat: 4.138370990753174 (4.140948967311692) + | > loss_mel: 20.957639694213867 (20.443783469822105) + | > loss_duration: 1.4087125062942505 (1.35146540807641) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.048444747924805 (30.418825315392535) + | > grad_norm_1: tensor(949.2124, device='cuda:0') (tensor(451.1358, device='cuda:0')) + | > current_lr_0: 0.00019875382047998183 + | > current_lr_1: 0.00019875382047998183 + | > step_time: 1.2701 (0.9411613858264424) + | > loader_time: 0.0236 (0.015469742235930067) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003717016666493517 (+0.00019355023160893842) + | > avg_loss_disc: 2.6215304110912565 (-0.027169354418490776) + | > avg_loss_disc_real_0: 0.16151953853191217 (-0.045058915431195085) + | > avg_loss_disc_real_1: 0.2861872797316694 (+0.05667542111366361) + | > avg_loss_disc_real_2: 0.21278656480160166 (-0.024693586724869737) + | > avg_loss_disc_real_3: 0.17969327848008337 (-0.05920088798441786) + | > avg_loss_disc_real_4: 0.21814754478474882 (+0.012283604829869371) + | > avg_loss_disc_real_5: 0.2373200753267775 (+0.024699470781265415) + | > avg_loss_0: 2.6215304110912565 (-0.027169354418490776) + | > avg_loss_gen: 1.9144085569584623 (-0.007596302539744304) + | > avg_loss_kl: 2.556134406556475 (-0.03322024548307434) + | > avg_loss_feat: 4.0113944449323276 (-0.08547814856184477) + | > avg_loss_mel: 20.278035305915996 (-0.06793914957249925) + | > avg_loss_duration: 1.3575695803824892 (-0.006147785389676752) + | > avg_loss_1: 30.117542145100046 (-0.20038162393772652) + + + > EPOCH: 52/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 11:05:58)  + + --> TIME: 2024-06-25 11:06:07 -- STEP: 9/591 -- GLOBAL_STEP: 30150 + | > loss_disc: 2.7115511894226074 (2.6336410310533314) + | > loss_disc_real_0: 0.12603983283042908 (0.1995840867360433) + | > loss_disc_real_1: 0.20477813482284546 (0.22133926219410366) + | > loss_disc_real_2: 0.20061665773391724 (0.22887544830640158) + | > loss_disc_real_3: 0.2900956869125366 (0.24044886065853965) + | > loss_disc_real_4: 0.24628391861915588 (0.21917969485123953) + | > loss_disc_real_5: 0.27499115467071533 (0.24034683075216082) + | > loss_0: 2.7115511894226074 (2.6336410310533314) + | > grad_norm_0: tensor(15.4836, device='cuda:0') (tensor(15.9073, device='cuda:0')) + | > loss_gen: 2.161324977874756 (2.0875502824783325) + | > loss_kl: 2.682704448699951 (2.682253811094496) + | > loss_feat: 4.630788803100586 (4.613365279303657) + | > loss_mel: 20.36067008972168 (20.68286175198025) + | > loss_duration: 1.2776732444763184 (1.315757950146993) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.113162994384766 (31.381789101494682) + | > grad_norm_1: tensor(120.3682, device='cuda:0') (tensor(758.6017, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.7868 (0.7765140798356798) + | > loader_time: 0.0097 (0.008692926830715604) + + + --> TIME: 2024-06-25 11:06:27 -- STEP: 34/591 -- GLOBAL_STEP: 30175 + | > loss_disc: 2.701185703277588 (2.648022371179917) + | > loss_disc_real_0: 0.2560603618621826 (0.20133503657930038) + | > loss_disc_real_1: 0.19778496026992798 (0.22179140194373972) + | > loss_disc_real_2: 0.20703519880771637 (0.23021886716870701) + | > loss_disc_real_3: 0.19812317192554474 (0.23432213900720372) + | > loss_disc_real_4: 0.19930456578731537 (0.2240829537896549) + | > loss_disc_real_5: 0.2036852389574051 (0.23781831913134632) + | > loss_0: 2.701185703277588 (2.648022371179917) + | > grad_norm_0: tensor(9.2849, device='cuda:0') (tensor(13.8981, device='cuda:0')) + | > loss_gen: 1.8881323337554932 (2.0231720910352817) + | > loss_kl: 2.4829349517822266 (2.6280412744073307) + | > loss_feat: 4.351013660430908 (4.419697614277111) + | > loss_mel: 20.939952850341797 (20.607710613923913) + | > loss_duration: 1.3481550216674805 (1.317442452206331) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.010189056396484 (30.996064129997702) + | > grad_norm_1: tensor(58.8552, device='cuda:0') (tensor(602.1260, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.7931 (0.7882209876004387) + | > loader_time: 0.0093 (0.009163512903101304) + + + --> TIME: 2024-06-25 11:06:47 -- STEP: 59/591 -- GLOBAL_STEP: 30200 + | > loss_disc: 2.702709436416626 (2.652875007209132) + | > loss_disc_real_0: 0.21960684657096863 (0.19968947591417927) + | > loss_disc_real_1: 0.248208686709404 (0.222343931258735) + | > loss_disc_real_2: 0.24311181902885437 (0.2315962468668566) + | > loss_disc_real_3: 0.2334994524717331 (0.23355386191505498) + | > loss_disc_real_4: 0.1969570368528366 (0.22497513183092666) + | > loss_disc_real_5: 0.2282988280057907 (0.23852650770696543) + | > loss_0: 2.702709436416626 (2.652875007209132) + | > grad_norm_0: tensor(12.5426, device='cuda:0') (tensor(13.5219, device='cuda:0')) + | > loss_gen: 2.026261329650879 (2.0173794899956645) + | > loss_kl: 2.3672549724578857 (2.5817461619942876) + | > loss_feat: 4.195218563079834 (4.390977839292109) + | > loss_mel: 20.138763427734375 (20.65366269774357) + | > loss_duration: 1.3166542053222656 (1.3265648494332523) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.044151306152344 (30.970330965721) + | > grad_norm_1: tensor(1002.9833, device='cuda:0') (tensor(595.4150, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.8159 (0.7969079785427805) + | > loader_time: 0.0111 (0.009503312030081021) + + + --> TIME: 2024-06-25 11:07:08 -- STEP: 84/591 -- GLOBAL_STEP: 30225 + | > loss_disc: 2.703442335128784 (2.6517712899616797) + | > loss_disc_real_0: 0.21332581341266632 (0.19848875027327312) + | > loss_disc_real_1: 0.18053960800170898 (0.222704380041077) + | > loss_disc_real_2: 0.2723166346549988 (0.2320909487704436) + | > loss_disc_real_3: 0.21028609573841095 (0.23432252822177752) + | > loss_disc_real_4: 0.2870035469532013 (0.22568237586390405) + | > loss_disc_real_5: 0.2585163712501526 (0.23881505500702632) + | > loss_0: 2.703442335128784 (2.6517712899616797) + | > grad_norm_0: tensor(10.9083, device='cuda:0') (tensor(13.2841, device='cuda:0')) + | > loss_gen: 1.9120291471481323 (2.024480867953527) + | > loss_kl: 2.4215245246887207 (2.559717615445455) + | > loss_feat: 4.509624481201172 (4.375506548654466) + | > loss_mel: 20.79005241394043 (20.640046914418544) + | > loss_duration: 1.315401554107666 (1.3335658680825007) + | > amp_scaler: 128.0 (129.5238095238095) + | > loss_1: 30.948631286621094 (30.933317774818057) + | > grad_norm_1: tensor(996.7529, device='cuda:0') (tensor(632.3946, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.8437 (0.8049034249214897) + | > loader_time: 0.0121 (0.00995448089781262) + + + --> TIME: 2024-06-25 11:07:30 -- STEP: 109/591 -- GLOBAL_STEP: 30250 + | > loss_disc: 2.618727207183838 (2.6600436792461166) + | > loss_disc_real_0: 0.16876015067100525 (0.19941363087214461) + | > loss_disc_real_1: 0.20632082223892212 (0.2230467402606929) + | > loss_disc_real_2: 0.21076180040836334 (0.23232775994943916) + | > loss_disc_real_3: 0.2723889648914337 (0.2351070269258744) + | > loss_disc_real_4: 0.2077581286430359 (0.22634135760845395) + | > loss_disc_real_5: 0.2314785271883011 (0.23852961913708154) + | > loss_0: 2.618727207183838 (2.6600436792461166) + | > grad_norm_0: tensor(8.6351, device='cuda:0') (tensor(13.5845, device='cuda:0')) + | > loss_gen: 2.0168235301971436 (2.014019284773309) + | > loss_kl: 2.6187570095062256 (2.546972259468988) + | > loss_feat: 4.502492904663086 (4.3411188847428095) + | > loss_mel: 20.884990692138672 (20.6145220590294) + | > loss_duration: 1.366432785987854 (1.338840122616619) + | > amp_scaler: 128.0 (129.1743119266054) + | > loss_1: 31.389495849609375 (30.855472599694487) + | > grad_norm_1: tensor(457.3005, device='cuda:0') (tensor(598.4098, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.8357 (0.8118434280430503) + | > loader_time: 0.0107 (0.010323826326142758) + + + --> TIME: 2024-06-25 11:07:51 -- STEP: 134/591 -- GLOBAL_STEP: 30275 + | > loss_disc: 2.667795181274414 (2.6589050631024955) + | > loss_disc_real_0: 0.19968023896217346 (0.19899585937608533) + | > loss_disc_real_1: 0.2014019638299942 (0.22315002132707568) + | > loss_disc_real_2: 0.20642459392547607 (0.2321493287584675) + | > loss_disc_real_3: 0.19023336470127106 (0.2345936381772383) + | > loss_disc_real_4: 0.18592900037765503 (0.22695858098232924) + | > loss_disc_real_5: 0.235026016831398 (0.23857325815887592) + | > loss_0: 2.667795181274414 (2.6589050631024955) + | > grad_norm_0: tensor(5.2783, device='cuda:0') (tensor(12.7726, device='cuda:0')) + | > loss_gen: 1.8522303104400635 (2.0097782754186366) + | > loss_kl: 2.4600143432617188 (2.536493344093436) + | > loss_feat: 4.248356819152832 (4.330598590978938) + | > loss_mel: 20.112773895263672 (20.601444913380185) + | > loss_duration: 1.3273051977157593 (1.3393383061707909) + | > amp_scaler: 128.0 (128.95522388059695) + | > loss_1: 30.00067901611328 (30.81765339979485) + | > grad_norm_1: tensor(180.9829, device='cuda:0') (tensor(560.9092, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.8264 (0.8175481949279556) + | > loader_time: 0.0116 (0.010578963293958066) + + + --> TIME: 2024-06-25 11:08:13 -- STEP: 159/591 -- GLOBAL_STEP: 30300 + | > loss_disc: 2.753798484802246 (2.6616126351386504) + | > loss_disc_real_0: 0.24200063943862915 (0.1997489082925724) + | > loss_disc_real_1: 0.21166540682315826 (0.22335600843594508) + | > loss_disc_real_2: 0.2668796479701996 (0.23258191319006793) + | > loss_disc_real_3: 0.26350298523902893 (0.2348035252506628) + | > loss_disc_real_4: 0.23734493553638458 (0.2270218568777888) + | > loss_disc_real_5: 0.2608022093772888 (0.23847644490265996) + | > loss_0: 2.753798484802246 (2.6616126351386504) + | > grad_norm_0: tensor(7.0101, device='cuda:0') (tensor(12.7079, device='cuda:0')) + | > loss_gen: 1.8375489711761475 (2.008787007451806) + | > loss_kl: 2.6015512943267822 (2.5296128380973384) + | > loss_feat: 4.041996002197266 (4.3205656000653185) + | > loss_mel: 20.01148796081543 (20.57023330304609) + | > loss_duration: 1.3378517627716064 (1.3378190439452164) + | > amp_scaler: 128.0 (128.8050314465407) + | > loss_1: 29.83043670654297 (30.767017772362667) + | > grad_norm_1: tensor(125.3266, device='cuda:0') (tensor(556.4167, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.8698 (0.8237490983879041) + | > loader_time: 0.0135 (0.010830105475659647) + + + --> TIME: 2024-06-25 11:08:35 -- STEP: 184/591 -- GLOBAL_STEP: 30325 + | > loss_disc: 2.6762917041778564 (2.6633349255375256) + | > loss_disc_real_0: 0.14473995566368103 (0.19953471778527546) + | > loss_disc_real_1: 0.2213837206363678 (0.22361848252299038) + | > loss_disc_real_2: 0.23668068647384644 (0.23267466259067474) + | > loss_disc_real_3: 0.2191416174173355 (0.23485854219483293) + | > loss_disc_real_4: 0.20434455573558807 (0.2270351369743762) + | > loss_disc_real_5: 0.2479473054409027 (0.238586169384096) + | > loss_0: 2.6762917041778564 (2.6633349255375256) + | > grad_norm_0: tensor(13.5515, device='cuda:0') (tensor(12.9895, device='cuda:0')) + | > loss_gen: 1.928139090538025 (2.007634534136108) + | > loss_kl: 2.445105791091919 (2.522205703932306) + | > loss_feat: 4.09188175201416 (4.31253748095554) + | > loss_mel: 21.291915893554688 (20.566646026528414) + | > loss_duration: 1.337073564529419 (1.3363595255043195) + | > amp_scaler: 128.0 (128.69565217391303) + | > loss_1: 31.0941162109375 (30.745383273000304) + | > grad_norm_1: tensor(574.7717, device='cuda:0') (tensor(555.3672, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.8761 (0.8289355104384215) + | > loader_time: 0.012 (0.01103422952734907) + + + --> TIME: 2024-06-25 11:08:57 -- STEP: 209/591 -- GLOBAL_STEP: 30350 + | > loss_disc: 2.6597254276275635 (2.6601247661992136) + | > loss_disc_real_0: 0.17981386184692383 (0.1991788111092371) + | > loss_disc_real_1: 0.20745745301246643 (0.22361625488580128) + | > loss_disc_real_2: 0.20066934823989868 (0.23234051437469191) + | > loss_disc_real_3: 0.23310643434524536 (0.23431147385442086) + | > loss_disc_real_4: 0.2308589667081833 (0.22680038199470373) + | > loss_disc_real_5: 0.23970697820186615 (0.2380910306883771) + | > loss_0: 2.6597254276275635 (2.6601247661992136) + | > grad_norm_0: tensor(12.8638, device='cuda:0') (tensor(12.9739, device='cuda:0')) + | > loss_gen: 1.9550291299819946 (2.007837788910387) + | > loss_kl: 2.6034443378448486 (2.520658567191311) + | > loss_feat: 4.1118669509887695 (4.308277898998351) + | > loss_mel: 20.38754653930664 (20.561744379655035) + | > loss_duration: 1.3048620223999023 (1.3358346885471253) + | > amp_scaler: 128.0 (128.61244019138752) + | > loss_1: 30.362747192382812 (30.73435332558372) + | > grad_norm_1: tensor(690.5687, device='cuda:0') (tensor(558.7893, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.8654 (0.8342653803848193) + | > loader_time: 0.0176 (0.011280820700540508) + + + --> TIME: 2024-06-25 11:09:20 -- STEP: 234/591 -- GLOBAL_STEP: 30375 + | > loss_disc: 2.60992431640625 (2.658004408208734) + | > loss_disc_real_0: 0.29138147830963135 (0.19908956215422374) + | > loss_disc_real_1: 0.24520519375801086 (0.22346444943776497) + | > loss_disc_real_2: 0.21466633677482605 (0.23264411321053138) + | > loss_disc_real_3: 0.23224197328090668 (0.23441850967132127) + | > loss_disc_real_4: 0.25611162185668945 (0.22577793303972635) + | > loss_disc_real_5: 0.2448393702507019 (0.23805756905140021) + | > loss_0: 2.60992431640625 (2.658004408208734) + | > grad_norm_0: tensor(28.4017, device='cuda:0') (tensor(13.1858, device='cuda:0')) + | > loss_gen: 2.034660577774048 (2.0127000660977816) + | > loss_kl: 2.4849045276641846 (2.5155467324786716) + | > loss_feat: 4.424643039703369 (4.315784160907451) + | > loss_mel: 20.40743637084961 (20.57481801611746) + | > loss_duration: 1.3564794063568115 (1.3357614034261458) + | > amp_scaler: 128.0 (128.5470085470086) + | > loss_1: 30.7081241607666 (30.754610379536945) + | > grad_norm_1: tensor(1160.0283, device='cuda:0') (tensor(580.9214, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.8799 (0.838862824643779) + | > loader_time: 0.016 (0.011580286881862552) + + + --> TIME: 2024-06-25 11:09:43 -- STEP: 259/591 -- GLOBAL_STEP: 30400 + | > loss_disc: 2.68865966796875 (2.661141721438258) + | > loss_disc_real_0: 0.1768350452184677 (0.1994863863041962) + | > loss_disc_real_1: 0.21915115416049957 (0.223615187993381) + | > loss_disc_real_2: 0.2079963982105255 (0.23271122914253514) + | > loss_disc_real_3: 0.20898275077342987 (0.2344220954363871) + | > loss_disc_real_4: 0.23050937056541443 (0.2262435105546561) + | > loss_disc_real_5: 0.2622373402118683 (0.23833092955088522) + | > loss_0: 2.68865966796875 (2.661141721438258) + | > grad_norm_0: tensor(7.8679, device='cuda:0') (tensor(13.3965, device='cuda:0')) + | > loss_gen: 1.8870798349380493 (2.0123114654916594) + | > loss_kl: 2.5153300762176514 (2.5144599119208495) + | > loss_feat: 4.0185465812683105 (4.306975871899874) + | > loss_mel: 20.27443504333496 (20.572175088536333) + | > loss_duration: 1.3610334396362305 (1.3359447579586368) + | > amp_scaler: 128.0 (128.49420849420855) + | > loss_1: 30.05642318725586 (30.74186709488681) + | > grad_norm_1: tensor(243.7887, device='cuda:0') (tensor(579.8128, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.8956 (0.8443258683193604) + | > loader_time: 0.0144 (0.01188618520051817) + + + --> TIME: 2024-06-25 11:10:06 -- STEP: 284/591 -- GLOBAL_STEP: 30425 + | > loss_disc: 2.679896593093872 (2.6611225185259975) + | > loss_disc_real_0: 0.2455885112285614 (0.19943077888497165) + | > loss_disc_real_1: 0.22252307832241058 (0.22404397551862285) + | > loss_disc_real_2: 0.255010724067688 (0.23277079151339933) + | > loss_disc_real_3: 0.21376876533031464 (0.2344686752578742) + | > loss_disc_real_4: 0.2686837315559387 (0.22578739815614593) + | > loss_disc_real_5: 0.2525583505630493 (0.23830545142712728) + | > loss_0: 2.679896593093872 (2.6611225185259975) + | > grad_norm_0: tensor(16.9031, device='cuda:0') (tensor(13.4535, device='cuda:0')) + | > loss_gen: 1.8939865827560425 (2.012090798414929) + | > loss_kl: 2.407355308532715 (2.5113088505368832) + | > loss_feat: 4.058940887451172 (4.297576534076476) + | > loss_mel: 20.5363826751709 (20.55549840524164) + | > loss_duration: 1.358633279800415 (1.3366955107366534) + | > amp_scaler: 128.0 (128.45070422535215) + | > loss_1: 30.25529670715332 (30.71317011873487) + | > grad_norm_1: tensor(826.3694, device='cuda:0') (tensor(595.2607, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.8932 (0.8491991439335781) + | > loader_time: 0.0158 (0.012166486659520115) + + + --> TIME: 2024-06-25 11:10:29 -- STEP: 309/591 -- GLOBAL_STEP: 30450 + | > loss_disc: 2.732130527496338 (2.6611906965187844) + | > loss_disc_real_0: 0.15048563480377197 (0.19966605705929416) + | > loss_disc_real_1: 0.2641896605491638 (0.22414860254738322) + | > loss_disc_real_2: 0.22164884209632874 (0.23264710465295416) + | > loss_disc_real_3: 0.24376945197582245 (0.23437489979089657) + | > loss_disc_real_4: 0.207632914185524 (0.22524269933067864) + | > loss_disc_real_5: 0.2398923635482788 (0.23843916329558226) + | > loss_0: 2.732130527496338 (2.6611906965187844) + | > grad_norm_0: tensor(12.6226, device='cuda:0') (tensor(13.9409, device='cuda:0')) + | > loss_gen: 1.9636180400848389 (2.0146366975068286) + | > loss_kl: 2.6222944259643555 (2.5103355017294784) + | > loss_feat: 4.408954620361328 (4.297152114145964) + | > loss_mel: 20.13956642150879 (20.538385977637038) + | > loss_duration: 1.3546600341796875 (1.3371622168902055) + | > amp_scaler: 128.0 (128.41423948220074) + | > loss_1: 30.489091873168945 (30.697672519869016) + | > grad_norm_1: tensor(140.0331, device='cuda:0') (tensor(617.5750, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.9151 (0.8547930115634953) + | > loader_time: 0.0158 (0.012430148603075156) + + + --> TIME: 2024-06-25 11:10:53 -- STEP: 334/591 -- GLOBAL_STEP: 30475 + | > loss_disc: 2.61372709274292 (2.6599293754486273) + | > loss_disc_real_0: 0.2691352367401123 (0.1997958404307593) + | > loss_disc_real_1: 0.2572217285633087 (0.2239814989134937) + | > loss_disc_real_2: 0.2273932844400406 (0.23259833642465624) + | > loss_disc_real_3: 0.2578485608100891 (0.23415012416725386) + | > loss_disc_real_4: 0.23978985846042633 (0.2250106499759023) + | > loss_disc_real_5: 0.2405884563922882 (0.23845887411675767) + | > loss_0: 2.61372709274292 (2.6599293754486273) + | > grad_norm_0: tensor(22.8117, device='cuda:0') (tensor(13.9935, device='cuda:0')) + | > loss_gen: 2.040616750717163 (2.0138180894766036) + | > loss_kl: 2.4511823654174805 (2.506409010487402) + | > loss_feat: 4.3278093338012695 (4.293094630726799) + | > loss_mel: 20.693063735961914 (20.537620407378604) + | > loss_duration: 1.3586277961730957 (1.3379442691802983) + | > amp_scaler: 128.0 (128.38323353293424) + | > loss_1: 30.871299743652344 (30.688886396899196) + | > grad_norm_1: tensor(1077.3234, device='cuda:0') (tensor(621.9091, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.9633 (0.8602562343289036) + | > loader_time: 0.0168 (0.012677013517139925) + + + --> TIME: 2024-06-25 11:11:17 -- STEP: 359/591 -- GLOBAL_STEP: 30500 + | > loss_disc: 2.7437984943389893 (2.6595437706014904) + | > loss_disc_real_0: 0.2935320734977722 (0.20006194083969564) + | > loss_disc_real_1: 0.21452584862709045 (0.2239054122260022) + | > loss_disc_real_2: 0.2599574029445648 (0.23267744743060267) + | > loss_disc_real_3: 0.26800209283828735 (0.23413087884197661) + | > loss_disc_real_4: 0.2516859173774719 (0.22451205925191015) + | > loss_disc_real_5: 0.25481465458869934 (0.2384104866088267) + | > loss_0: 2.7437984943389893 (2.6595437706014904) + | > grad_norm_0: tensor(22.1074, device='cuda:0') (tensor(14.2163, device='cuda:0')) + | > loss_gen: 1.7638072967529297 (2.0146461698670253) + | > loss_kl: 2.560812473297119 (2.5044122638808655) + | > loss_feat: 3.980599880218506 (4.289604768115501) + | > loss_mel: 20.379140853881836 (20.523941608524584) + | > loss_duration: 1.3376730680465698 (1.339110617186035) + | > amp_scaler: 128.0 (128.3565459610029) + | > loss_1: 30.02203369140625 (30.671715420268704) + | > grad_norm_1: tensor(739.3995, device='cuda:0') (tensor(634.5309, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.9507 (0.8659587789710849) + | > loader_time: 0.0179 (0.012931039406396562) + + + --> TIME: 2024-06-25 11:11:41 -- STEP: 384/591 -- GLOBAL_STEP: 30525 + | > loss_disc: 2.7959792613983154 (2.6598069773366064) + | > loss_disc_real_0: 0.15417908132076263 (0.199764694358843) + | > loss_disc_real_1: 0.2699543833732605 (0.22385665882999697) + | > loss_disc_real_2: 0.26806339621543884 (0.23268614837434143) + | > loss_disc_real_3: 0.22132182121276855 (0.23421710884819427) + | > loss_disc_real_4: 0.2404719740152359 (0.22470359100649753) + | > loss_disc_real_5: 0.26184070110321045 (0.23840139092256626) + | > loss_0: 2.7959792613983154 (2.6598069773366064) + | > grad_norm_0: tensor(15.2409, device='cuda:0') (tensor(14.1713, device='cuda:0')) + | > loss_gen: 2.026611328125 (2.0129881758863744) + | > loss_kl: 2.5658791065216064 (2.5048016688476005) + | > loss_feat: 3.9045345783233643 (4.278699622179074) + | > loss_mel: 19.882152557373047 (20.502284412582714) + | > loss_duration: 1.377772331237793 (1.3406457540889594) + | > amp_scaler: 128.0 (128.33333333333348) + | > loss_1: 29.75695037841797 (30.639419610301655) + | > grad_norm_1: tensor(594.2847, device='cuda:0') (tensor(631.2384, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.9696 (0.871882832298676) + | > loader_time: 0.0189 (0.013205523292223622) + + + --> TIME: 2024-06-25 11:12:06 -- STEP: 409/591 -- GLOBAL_STEP: 30550 + | > loss_disc: 2.584103584289551 (2.660941446322976) + | > loss_disc_real_0: 0.16383206844329834 (0.19997336048733044) + | > loss_disc_real_1: 0.25059595704078674 (0.22377414593341008) + | > loss_disc_real_2: 0.21883736550807953 (0.23261493553742218) + | > loss_disc_real_3: 0.2572947144508362 (0.23427953591119396) + | > loss_disc_real_4: 0.2195935994386673 (0.22520600732935087) + | > loss_disc_real_5: 0.21108046174049377 (0.23835674482044789) + | > loss_0: 2.584103584289551 (2.660941446322976) + | > grad_norm_0: tensor(10.9599, device='cuda:0') (tensor(14.1984, device='cuda:0')) + | > loss_gen: 2.1028854846954346 (2.010986204543733) + | > loss_kl: 2.4502954483032227 (2.5040162915414004) + | > loss_feat: 4.419929027557373 (4.270606578999454) + | > loss_mel: 20.19135093688965 (20.495158067250884) + | > loss_duration: 1.3311374187469482 (1.341438095843589) + | > amp_scaler: 128.0 (128.31295843520792) + | > loss_1: 30.49559783935547 (30.62220523060097) + | > grad_norm_1: tensor(659.6205, device='cuda:0') (tensor(619.9186, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 0.9718 (0.8777766484211591) + | > loader_time: 0.0194 (0.01349407827941597) + + + --> TIME: 2024-06-25 11:12:32 -- STEP: 434/591 -- GLOBAL_STEP: 30575 + | > loss_disc: 2.711712598800659 (2.661222574348275) + | > loss_disc_real_0: 0.22400271892547607 (0.19987156762299443) + | > loss_disc_real_1: 0.27599215507507324 (0.22374874477012915) + | > loss_disc_real_2: 0.25436854362487793 (0.23277488484772668) + | > loss_disc_real_3: 0.21808211505413055 (0.23426732259549304) + | > loss_disc_real_4: 0.25303179025650024 (0.2255669737809814) + | > loss_disc_real_5: 0.24865981936454773 (0.23846256314060105) + | > loss_0: 2.711712598800659 (2.661222574348275) + | > grad_norm_0: tensor(11.2524, device='cuda:0') (tensor(14.0243, device='cuda:0')) + | > loss_gen: 1.961359977722168 (2.008729525150792) + | > loss_kl: 2.515523672103882 (2.5030919467249224) + | > loss_feat: 4.205811977386475 (4.258797779061278) + | > loss_mel: 20.651935577392578 (20.489094105733695) + | > loss_duration: 1.3439645767211914 (1.3423810667156635) + | > amp_scaler: 128.0 (128.29493087557614) + | > loss_1: 30.67859649658203 (30.602094426133117) + | > grad_norm_1: tensor(706.1787, device='cuda:0') (tensor(610.6736, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 1.0036 (0.8846525761388961) + | > loader_time: 0.0167 (0.013782219952701982) + + + --> TIME: 2024-06-25 11:12:58 -- STEP: 459/591 -- GLOBAL_STEP: 30600 + | > loss_disc: 2.630418300628662 (2.660952629346994) + | > loss_disc_real_0: 0.23745472729206085 (0.19992183114773301) + | > loss_disc_real_1: 0.24730515480041504 (0.22356119450011286) + | > loss_disc_real_2: 0.2285155951976776 (0.23269391848760493) + | > loss_disc_real_3: 0.2518140375614166 (0.23427845521430304) + | > loss_disc_real_4: 0.21926362812519073 (0.2255230650532999) + | > loss_disc_real_5: 0.2770118713378906 (0.23849599258182874) + | > loss_0: 2.630418300628662 (2.660952629346994) + | > grad_norm_0: tensor(13.4764, device='cuda:0') (tensor(14.0770, device='cuda:0')) + | > loss_gen: 2.043595790863037 (2.0079334467844245) + | > loss_kl: 2.3965229988098145 (2.5004696908339956) + | > loss_feat: 4.315498352050781 (4.252517161026501) + | > loss_mel: 20.742380142211914 (20.48347687149878) + | > loss_duration: 1.3646775484085083 (1.343526120798781) + | > amp_scaler: 128.0 (128.27886710239662) + | > loss_1: 30.862674713134766 (30.587923290942488) + | > grad_norm_1: tensor(1081.2294, device='cuda:0') (tensor(608.8917, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 1.0075 (0.8918024283348863) + | > loader_time: 0.0174 (0.014058566041501786) + + + --> TIME: 2024-06-25 11:13:24 -- STEP: 484/591 -- GLOBAL_STEP: 30625 + | > loss_disc: 2.6455109119415283 (2.66255524335814) + | > loss_disc_real_0: 0.15580041706562042 (0.19988823520435275) + | > loss_disc_real_1: 0.23744860291481018 (0.22373022511601448) + | > loss_disc_real_2: 0.20638959109783173 (0.2327178020553648) + | > loss_disc_real_3: 0.24774041771888733 (0.23437514577892202) + | > loss_disc_real_4: 0.23045609891414642 (0.22567595932478748) + | > loss_disc_real_5: 0.23221711814403534 (0.23858577718050028) + | > loss_0: 2.6455109119415283 (2.66255524335814) + | > grad_norm_0: tensor(13.2843, device='cuda:0') (tensor(14.1625, device='cuda:0')) + | > loss_gen: 2.100032329559326 (2.0059397587598844) + | > loss_kl: 2.526590347290039 (2.496547339376339) + | > loss_feat: 4.087014675140381 (4.2387943947610776) + | > loss_mel: 20.29037857055664 (20.4812471098151) + | > loss_duration: 1.3694567680358887 (1.3448886984635986) + | > amp_scaler: 128.0 (128.26446280991743) + | > loss_1: 30.37347412109375 (30.567417318170722) + | > grad_norm_1: tensor(707.4227, device='cuda:0') (tensor(606.8697, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 1.0445 (0.8987369231941285) + | > loader_time: 0.0204 (0.0143628553910689) + + + --> TIME: 2024-06-25 11:13:51 -- STEP: 509/591 -- GLOBAL_STEP: 30650 + | > loss_disc: 2.6675267219543457 (2.663019532070648) + | > loss_disc_real_0: 0.16506929695606232 (0.20005329484614026) + | > loss_disc_real_1: 0.1967259645462036 (0.22377701642354017) + | > loss_disc_real_2: 0.20614556968212128 (0.23283686420069696) + | > loss_disc_real_3: 0.23600904643535614 (0.23445517547823824) + | > loss_disc_real_4: 0.20787084102630615 (0.22539864410234575) + | > loss_disc_real_5: 0.2491188794374466 (0.23877417476214688) + | > loss_0: 2.6675267219543457 (2.663019532070648) + | > grad_norm_0: tensor(16.6704, device='cuda:0') (tensor(14.2086, device='cuda:0')) + | > loss_gen: 2.0330934524536133 (2.0063555936213784) + | > loss_kl: 2.368194580078125 (2.4938639208001097) + | > loss_feat: 4.233839511871338 (4.234448949574019) + | > loss_mel: 20.998126983642578 (20.480776612557214) + | > loss_duration: 1.391862154006958 (1.3466496603428038) + | > amp_scaler: 128.0 (128.25147347740676) + | > loss_1: 31.025117874145508 (30.562094753992348) + | > grad_norm_1: tensor(839.6882, device='cuda:0') (tensor(615.3428, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 1.0505 (0.9070990427538078) + | > loader_time: 0.0232 (0.014661956161319163) + + + --> TIME: 2024-06-25 11:14:20 -- STEP: 534/591 -- GLOBAL_STEP: 30675 + | > loss_disc: 2.6746673583984375 (2.6628979024815655) + | > loss_disc_real_0: 0.26247072219848633 (0.20026989645335103) + | > loss_disc_real_1: 0.2534353733062744 (0.2238945390345452) + | > loss_disc_real_2: 0.2286459505558014 (0.23302276410190353) + | > loss_disc_real_3: 0.23774753510951996 (0.23443345853898417) + | > loss_disc_real_4: 0.20718026161193848 (0.22493725839290724) + | > loss_disc_real_5: 0.24774841964244843 (0.23880213423726265) + | > loss_0: 2.6746673583984375 (2.6628979024815655) + | > grad_norm_0: tensor(23.3414, device='cuda:0') (tensor(14.4674, device='cuda:0')) + | > loss_gen: 1.8452138900756836 (2.0071588417563984) + | > loss_kl: 2.4108543395996094 (2.492200534888421) + | > loss_feat: 3.9966893196105957 (4.2281813375958786) + | > loss_mel: 19.93798828125 (20.467211876915627) + | > loss_duration: 1.3728728294372559 (1.3484604680136356) + | > amp_scaler: 128.0 (128.2397003745319) + | > loss_1: 29.56361961364746 (30.543213097790208) + | > grad_norm_1: tensor(1143.4120, device='cuda:0') (tensor(626.4973, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 1.1278 (0.9170351421341912) + | > loader_time: 0.0182 (0.01498206188616235) + + + --> TIME: 2024-06-25 11:14:50 -- STEP: 559/591 -- GLOBAL_STEP: 30700 + | > loss_disc: 2.6806998252868652 (2.6639823209834237) + | > loss_disc_real_0: 0.14507266879081726 (0.19998461391930072) + | > loss_disc_real_1: 0.24279671907424927 (0.2240044131528499) + | > loss_disc_real_2: 0.21954286098480225 (0.23307205462071892) + | > loss_disc_real_3: 0.22203928232192993 (0.23446956980633613) + | > loss_disc_real_4: 0.25538867712020874 (0.22529001549979227) + | > loss_disc_real_5: 0.24379979074001312 (0.23882638309850676) + | > loss_0: 2.6806998252868652 (2.6639823209834237) + | > grad_norm_0: tensor(14.1142, device='cuda:0') (tensor(14.4826, device='cuda:0')) + | > loss_gen: 2.0815813541412354 (2.0064512559467644) + | > loss_kl: 2.4765334129333496 (2.491285395750206) + | > loss_feat: 4.14268159866333 (4.21704820742121) + | > loss_mel: 20.57352638244629 (20.452374500963884) + | > loss_duration: 1.4093362092971802 (1.3502649753379485) + | > amp_scaler: 128.0 (128.22898032200365) + | > loss_1: 30.683658599853516 (30.517424370180514) + | > grad_norm_1: tensor(592.1656, device='cuda:0') (tensor(623.8411, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 1.1409 (0.9279298978542813) + | > loader_time: 0.0253 (0.015316805387438947) + + + --> TIME: 2024-06-25 11:15:21 -- STEP: 584/591 -- GLOBAL_STEP: 30725 + | > loss_disc: 2.588285446166992 (2.6642678522900365) + | > loss_disc_real_0: 0.2169814109802246 (0.2000808658410016) + | > loss_disc_real_1: 0.22964784502983093 (0.22408107420659218) + | > loss_disc_real_2: 0.25627458095550537 (0.2330839001662927) + | > loss_disc_real_3: 0.2335980385541916 (0.2343077984931943) + | > loss_disc_real_4: 0.19708296656608582 (0.2254378416460671) + | > loss_disc_real_5: 0.20795200765132904 (0.23870677705088708) + | > loss_0: 2.588285446166992 (2.6642678522900365) + | > grad_norm_0: tensor(9.2194, device='cuda:0') (tensor(14.4178, device='cuda:0')) + | > loss_gen: 2.0120856761932373 (2.0043841769842246) + | > loss_kl: 2.461155652999878 (2.490513619494764) + | > loss_feat: 4.078791618347168 (4.205370621321953) + | > loss_mel: 20.056779861450195 (20.440690631735794) + | > loss_duration: 1.387312650680542 (1.3519915144329206) + | > amp_scaler: 128.0 (128.21917808219192) + | > loss_1: 29.996124267578125 (30.49295057335945) + | > grad_norm_1: tensor(408.1637, device='cuda:0') (tensor(613.1307, device='cuda:0')) + | > current_lr_0: 0.00019872897625242182 + | > current_lr_1: 0.00019872897625242182 + | > step_time: 1.262 (0.940517448399165) + | > loader_time: 0.027 (0.01571202400612505) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035775012158332986 (-0.00013951545066021856) + | > avg_loss_disc: 2.70050412036003 (+0.07897370926877345) + | > avg_loss_disc_real_0: 0.199567105858884 (+0.038047567326971815) + | > avg_loss_disc_real_1: 0.22796084050168383 (-0.05822643922998558) + | > avg_loss_disc_real_2: 0.22242804505723587 (+0.009641480255634216) + | > avg_loss_disc_real_3: 0.20519271366139677 (+0.025499435181313396) + | > avg_loss_disc_real_4: 0.24584018328088394 (+0.027692638496135125) + | > avg_loss_disc_real_5: 0.25448146589258874 (+0.01716139056581123) + | > avg_loss_0: 2.70050412036003 (+0.07897370926877345) + | > avg_loss_gen: 1.8615911843928885 (-0.05281737256557384) + | > avg_loss_kl: 2.604942651505165 (+0.04880824494868996) + | > avg_loss_feat: 3.964759035313382 (-0.046635409618945634) + | > avg_loss_mel: 20.544480060009246 (+0.26644475409324997) + | > avg_loss_duration: 1.3865660581182933 (+0.02899647773580405) + | > avg_loss_1: 30.36233873570219 (+0.24479659060214232) + + + > EPOCH: 53/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 11:15:37)  + + --> TIME: 2024-06-25 11:15:53 -- STEP: 18/591 -- GLOBAL_STEP: 30750 + | > loss_disc: 2.638463020324707 (2.6674405998653836) + | > loss_disc_real_0: 0.14728450775146484 (0.20368268506394493) + | > loss_disc_real_1: 0.20762844383716583 (0.22274045480622184) + | > loss_disc_real_2: 0.25675302743911743 (0.23569496472676596) + | > loss_disc_real_3: 0.23672395944595337 (0.23365740146901873) + | > loss_disc_real_4: 0.2321261167526245 (0.22542945212788051) + | > loss_disc_real_5: 0.24122531712055206 (0.23801274018155205) + | > loss_0: 2.638463020324707 (2.6674405998653836) + | > grad_norm_0: tensor(22.6707, device='cuda:0') (tensor(14.3331, device='cuda:0')) + | > loss_gen: 2.1949217319488525 (2.0501010947757297) + | > loss_kl: 2.5018436908721924 (2.647957788573371) + | > loss_feat: 4.437343120574951 (4.515346792009142) + | > loss_mel: 20.321651458740234 (20.658417701721188) + | > loss_duration: 1.2986420392990112 (1.3153750830226474) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.7544002532959 (31.187198003133137) + | > grad_norm_1: tensor(931.4710, device='cuda:0') (tensor(694.0031, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.7711 (0.773574259546068) + | > loader_time: 0.0088 (0.008980062272813585) + + + --> TIME: 2024-06-25 11:16:13 -- STEP: 43/591 -- GLOBAL_STEP: 30775 + | > loss_disc: 2.6151669025421143 (2.6716399026471516) + | > loss_disc_real_0: 0.22391262650489807 (0.2061508405347203) + | > loss_disc_real_1: 0.264136403799057 (0.22173554883446805) + | > loss_disc_real_2: 0.22712956368923187 (0.23624945172043735) + | > loss_disc_real_3: 0.25005054473876953 (0.23337186075920283) + | > loss_disc_real_4: 0.28356245160102844 (0.22466154604457145) + | > loss_disc_real_5: 0.21557410061359406 (0.2375350722739863) + | > loss_0: 2.6151669025421143 (2.6716399026471516) + | > grad_norm_0: tensor(12.7750, device='cuda:0') (tensor(14.8739, device='cuda:0')) + | > loss_gen: 2.1764025688171387 (2.0273099472356395) + | > loss_kl: 2.530540943145752 (2.599037641702696) + | > loss_feat: 4.354180812835693 (4.412856101989747) + | > loss_mel: 21.231929779052734 (20.625984147537586) + | > loss_duration: 1.3536372184753418 (1.3239269228868709) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.646692276000977 (30.98911458392476) + | > grad_norm_1: tensor(969.7383, device='cuda:0') (tensor(672.6537, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.7948 (0.7861489972402882) + | > loader_time: 0.0113 (0.009603561357010246) + + + --> TIME: 2024-06-25 11:16:34 -- STEP: 68/591 -- GLOBAL_STEP: 30800 + | > loss_disc: 2.6339759826660156 (2.6702348239281597) + | > loss_disc_real_0: 0.17929698526859283 (0.2026534014764954) + | > loss_disc_real_1: 0.21152134239673615 (0.22262497005217216) + | > loss_disc_real_2: 0.2230629324913025 (0.23713343809632695) + | > loss_disc_real_3: 0.20007500052452087 (0.2335779921096914) + | > loss_disc_real_4: 0.24776412546634674 (0.22717708086266236) + | > loss_disc_real_5: 0.22816012799739838 (0.23776465172276778) + | > loss_0: 2.6339759826660156 (2.6702348239281597) + | > grad_norm_0: tensor(4.3255, device='cuda:0') (tensor(13.2018, device='cuda:0')) + | > loss_gen: 1.9052257537841797 (2.01382434894057) + | > loss_kl: 2.5430078506469727 (2.5781791525728557) + | > loss_feat: 4.166312217712402 (4.348807517219992) + | > loss_mel: 20.78764533996582 (20.60747253193575) + | > loss_duration: 1.347215175628662 (1.3304492901353275) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.749406814575195 (30.878732793471393) + | > grad_norm_1: tensor(95.3742, device='cuda:0') (tensor(561.4895, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.8118 (0.7963100251029518) + | > loader_time: 0.0104 (0.009941244826597329) + + + --> TIME: 2024-06-25 11:16:55 -- STEP: 93/591 -- GLOBAL_STEP: 30825 + | > loss_disc: 2.6922411918640137 (2.6724394316314375) + | > loss_disc_real_0: 0.21258974075317383 (0.20266911123068101) + | > loss_disc_real_1: 0.23795153200626373 (0.22270605705117666) + | > loss_disc_real_2: 0.25249868631362915 (0.23552505191295378) + | > loss_disc_real_3: 0.23191353678703308 (0.23303545450651517) + | > loss_disc_real_4: 0.2337118536233902 (0.2291781557824022) + | > loss_disc_real_5: 0.24047690629959106 (0.2384056322356706) + | > loss_0: 2.6922411918640137 (2.6724394316314375) + | > grad_norm_0: tensor(14.5989, device='cuda:0') (tensor(13.3480, device='cuda:0')) + | > loss_gen: 1.8759678602218628 (2.0081105219420574) + | > loss_kl: 2.3022308349609375 (2.5638203928547516) + | > loss_feat: 4.068931579589844 (4.334064001678136) + | > loss_mel: 19.63408851623535 (20.61899201587964) + | > loss_duration: 1.3783670663833618 (1.3377154821990638) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.259584426879883 (30.862702277398878) + | > grad_norm_1: tensor(461.4435, device='cuda:0') (tensor(485.6497, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.8369 (0.8039506558449038) + | > loader_time: 0.0104 (0.01025026075301632) + + + --> TIME: 2024-06-25 11:17:16 -- STEP: 118/591 -- GLOBAL_STEP: 30850 + | > loss_disc: 2.6438708305358887 (2.673361602476088) + | > loss_disc_real_0: 0.15220296382904053 (0.20143706547254223) + | > loss_disc_real_1: 0.19408218562602997 (0.22315266185392768) + | > loss_disc_real_2: 0.22323893010616302 (0.2357376259514841) + | > loss_disc_real_3: 0.2113044708967209 (0.2327386611851595) + | > loss_disc_real_4: 0.2066357284784317 (0.22947346583261327) + | > loss_disc_real_5: 0.23083406686782837 (0.2389070963708021) + | > loss_0: 2.6438708305358887 (2.673361602476088) + | > grad_norm_0: tensor(15.8908, device='cuda:0') (tensor(12.5081, device='cuda:0')) + | > loss_gen: 2.0449209213256836 (2.001381888227949) + | > loss_kl: 2.452728033065796 (2.5512634010638213) + | > loss_feat: 4.356186866760254 (4.30443322456489) + | > loss_mel: 20.52566909790039 (20.616570585865084) + | > loss_duration: 1.352583408355713 (1.338915933996944) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.732088088989258 (30.812564833689546) + | > grad_norm_1: tensor(46.7378, device='cuda:0') (tensor(444.8507, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.8503 (0.8113740419937392) + | > loader_time: 0.012 (0.010485071246906862) + + + --> TIME: 2024-06-25 11:17:38 -- STEP: 143/591 -- GLOBAL_STEP: 30875 + | > loss_disc: 2.7058310508728027 (2.6713932494183528) + | > loss_disc_real_0: 0.24049994349479675 (0.20204796230459546) + | > loss_disc_real_1: 0.28195130825042725 (0.224013339702066) + | > loss_disc_real_2: 0.2796987295150757 (0.2355790190346591) + | > loss_disc_real_3: 0.2396651953458786 (0.23272247987610478) + | > loss_disc_real_4: 0.23775053024291992 (0.22811935998343089) + | > loss_disc_real_5: 0.23647217452526093 (0.2383474415832466) + | > loss_0: 2.7058310508728027 (2.6713932494183528) + | > grad_norm_0: tensor(10.8860, device='cuda:0') (tensor(13.0588, device='cuda:0')) + | > loss_gen: 1.966570496559143 (2.006578311219917) + | > loss_kl: 2.3629956245422363 (2.5418767979095023) + | > loss_feat: 4.294292449951172 (4.309750203486089) + | > loss_mel: 20.363086700439453 (20.59646901217374) + | > loss_duration: 1.314279556274414 (1.3388233484921757) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.301223754882812 (30.793497539066767) + | > grad_norm_1: tensor(492.9016, device='cuda:0') (tensor(481.5331, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.8554 (0.8172834919882822) + | > loader_time: 0.0125 (0.010668070999892438) + + + --> TIME: 2024-06-25 11:18:00 -- STEP: 168/591 -- GLOBAL_STEP: 30900 + | > loss_disc: 2.7297873497009277 (2.670024328288579) + | > loss_disc_real_0: 0.23757968842983246 (0.20128258503973484) + | > loss_disc_real_1: 0.23172207176685333 (0.22370218689597787) + | > loss_disc_real_2: 0.23524002730846405 (0.23545810908433937) + | > loss_disc_real_3: 0.28238070011138916 (0.23287855177408173) + | > loss_disc_real_4: 0.25923487544059753 (0.2277582436800003) + | > loss_disc_real_5: 0.2578185796737671 (0.2384787735839685) + | > loss_0: 2.7297873497009277 (2.670024328288579) + | > grad_norm_0: tensor(18.8642, device='cuda:0') (tensor(13.2838, device='cuda:0')) + | > loss_gen: 2.10107421875 (2.0082026159479534) + | > loss_kl: 2.352668523788452 (2.533632847524823) + | > loss_feat: 4.403507709503174 (4.310222540582929) + | > loss_mel: 21.001750946044922 (20.57035613059997) + | > loss_duration: 1.319122314453125 (1.3372542482046856) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.178123474121094 (30.759668282100133) + | > grad_norm_1: tensor(1134.7787, device='cuda:0') (tensor(510.1326, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.8609 (0.824439796663466) + | > loader_time: 0.0117 (0.010862614427294045) + + + --> TIME: 2024-06-25 11:18:22 -- STEP: 193/591 -- GLOBAL_STEP: 30925 + | > loss_disc: 2.6829206943511963 (2.6668765260765603) + | > loss_disc_real_0: 0.18195579946041107 (0.20074091071909572) + | > loss_disc_real_1: 0.21632815897464752 (0.22370918535197953) + | > loss_disc_real_2: 0.20509189367294312 (0.23502813325953606) + | > loss_disc_real_3: 0.2107795774936676 (0.23302777744636635) + | > loss_disc_real_4: 0.22299106419086456 (0.22642166657769) + | > loss_disc_real_5: 0.2389192283153534 (0.2383909267928316) + | > loss_0: 2.6829206943511963 (2.6668765260765603) + | > grad_norm_0: tensor(9.4365, device='cuda:0') (tensor(13.4870, device='cuda:0')) + | > loss_gen: 1.999548077583313 (2.010879743284514) + | > loss_kl: 2.668956756591797 (2.534140887038076) + | > loss_feat: 4.382978439331055 (4.313268762796038) + | > loss_mel: 20.73725128173828 (20.560560196792515) + | > loss_duration: 1.3070694208145142 (1.336466979486337) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.09580421447754 (30.755316472424127) + | > grad_norm_1: tensor(688.5819, device='cuda:0') (tensor(539.0588, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.8791 (0.8297908726133831) + | > loader_time: 0.0136 (0.011067208848468998) + + + --> TIME: 2024-06-25 11:18:44 -- STEP: 218/591 -- GLOBAL_STEP: 30950 + | > loss_disc: 2.616973400115967 (2.664118438685707) + | > loss_disc_real_0: 0.19423344731330872 (0.20026683062314987) + | > loss_disc_real_1: 0.26551008224487305 (0.22356219370977595) + | > loss_disc_real_2: 0.22996091842651367 (0.23479808374829247) + | > loss_disc_real_3: 0.22326932847499847 (0.23314214026162383) + | > loss_disc_real_4: 0.22023378312587738 (0.22630540069636948) + | > loss_disc_real_5: 0.2429073601961136 (0.2381554552858029) + | > loss_0: 2.616973400115967 (2.664118438685707) + | > grad_norm_0: tensor(16.2652, device='cuda:0') (tensor(13.8430, device='cuda:0')) + | > loss_gen: 2.171508312225342 (2.013651223357666) + | > loss_kl: 2.4492387771606445 (2.5282697622929127) + | > loss_feat: 4.4453444480896 (4.310677295431086) + | > loss_mel: 20.686296463012695 (20.56063872521077) + | > loss_duration: 1.3198944330215454 (1.3356550888184013) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.072282791137695 (30.74889200543045) + | > grad_norm_1: tensor(1159.0574, device='cuda:0') (tensor(560.5867, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.8704 (0.8345502921200674) + | > loader_time: 0.0153 (0.011369687701584003) + + + --> TIME: 2024-06-25 11:19:07 -- STEP: 243/591 -- GLOBAL_STEP: 30975 + | > loss_disc: 2.68076491355896 (2.660895921565869) + | > loss_disc_real_0: 0.13770648837089539 (0.19974784795271516) + | > loss_disc_real_1: 0.2653309404850006 (0.22326391518361283) + | > loss_disc_real_2: 0.24091044068336487 (0.23416027978614526) + | > loss_disc_real_3: 0.23539488017559052 (0.23321175121476131) + | > loss_disc_real_4: 0.25202202796936035 (0.22614485920701988) + | > loss_disc_real_5: 0.23150373995304108 (0.23785952941625696) + | > loss_0: 2.68076491355896 (2.660895921565869) + | > grad_norm_0: tensor(19.6379, device='cuda:0') (tensor(13.9350, device='cuda:0')) + | > loss_gen: 2.1140193939208984 (2.013774443555762) + | > loss_kl: 2.5314559936523438 (2.525829576170492) + | > loss_feat: 4.339657306671143 (4.316294606330464) + | > loss_mel: 20.525453567504883 (20.561053970713676) + | > loss_duration: 1.3283263444900513 (1.3361033971417589) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.838912963867188 (30.753055925722474) + | > grad_norm_1: tensor(404.4484, device='cuda:0') (tensor(569.6295, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.8844 (0.8398875228662059) + | > loader_time: 0.0151 (0.011691470205047984) + + + --> TIME: 2024-06-25 11:19:30 -- STEP: 268/591 -- GLOBAL_STEP: 31000 + | > loss_disc: 2.625962495803833 (2.6604044704294925) + | > loss_disc_real_0: 0.2863568067550659 (0.1999195694589793) + | > loss_disc_real_1: 0.1930367350578308 (0.22331007469945877) + | > loss_disc_real_2: 0.20958755910396576 (0.23377841827807141) + | > loss_disc_real_3: 0.180130735039711 (0.23313382201230348) + | > loss_disc_real_4: 0.16783009469509125 (0.22530909683277361) + | > loss_disc_real_5: 0.2248087376356125 (0.23790996143622184) + | > loss_0: 2.625962495803833 (2.6604044704294925) + | > grad_norm_0: tensor(21.6055, device='cuda:0') (tensor(14.2030, device='cuda:0')) + | > loss_gen: 2.2624363899230957 (2.0155909759784816) + | > loss_kl: 2.5123958587646484 (2.524879081925348) + | > loss_feat: 4.5815253257751465 (4.3191362450371935) + | > loss_mel: 20.693185806274414 (20.554623169685478) + | > loss_duration: 1.309295415878296 (1.3360880146275707) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.35883903503418 (30.750317452558832) + | > grad_norm_1: tensor(1267.7151, device='cuda:0') (tensor(585.8234, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.9076 (0.8452813598647047) + | > loader_time: 0.0149 (0.012004813151573068) + + + --> TIME: 2024-06-25 11:19:53 -- STEP: 293/591 -- GLOBAL_STEP: 31025 + | > loss_disc: 2.620363712310791 (2.6603427511026436) + | > loss_disc_real_0: 0.14975200593471527 (0.19973318444075439) + | > loss_disc_real_1: 0.19717545807361603 (0.223825518588564) + | > loss_disc_real_2: 0.22715739905834198 (0.23363142030841255) + | > loss_disc_real_3: 0.20532943308353424 (0.23361358478817923) + | > loss_disc_real_4: 0.23775744438171387 (0.2252255796165597) + | > loss_disc_real_5: 0.2559169828891754 (0.23813940091344685) + | > loss_0: 2.620363712310791 (2.6603427511026436) + | > grad_norm_0: tensor(21.1036, device='cuda:0') (tensor(14.4427, device='cuda:0')) + | > loss_gen: 2.075835943222046 (2.0170664795429647) + | > loss_kl: 2.5779411792755127 (2.5224337626642734) + | > loss_feat: 4.394953727722168 (4.3172240867549645) + | > loss_mel: 20.55350685119629 (20.542428560224415) + | > loss_duration: 1.3435543775558472 (1.3367481036397786) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.945791244506836 (30.73590093749375) + | > grad_norm_1: tensor(1011.1924, device='cuda:0') (tensor(605.6247, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.926 (0.8506390763631044) + | > loader_time: 0.015 (0.01229337864768383) + + + --> TIME: 2024-06-25 11:20:16 -- STEP: 318/591 -- GLOBAL_STEP: 31050 + | > loss_disc: 2.7977404594421387 (2.658263330189688) + | > loss_disc_real_0: 0.1678435057401657 (0.1998190014570389) + | > loss_disc_real_1: 0.21882925927639008 (0.22367078553205758) + | > loss_disc_real_2: 0.2508818507194519 (0.23353747718926496) + | > loss_disc_real_3: 0.2482142448425293 (0.23379090015993179) + | > loss_disc_real_4: 0.2038678228855133 (0.2236953659346269) + | > loss_disc_real_5: 0.2702469229698181 (0.23803756381355742) + | > loss_0: 2.7977404594421387 (2.658263330189688) + | > grad_norm_0: tensor(7.4349, device='cuda:0') (tensor(14.8004, device='cuda:0')) + | > loss_gen: 1.9749661684036255 (2.021641938566414) + | > loss_kl: 2.4143576622009277 (2.51894731941463) + | > loss_feat: 3.9238524436950684 (4.3205622007262) + | > loss_mel: 20.73302459716797 (20.54354473329941) + | > loss_duration: 1.3635315895080566 (1.3370837651708594) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.409732818603516 (30.741779921189796) + | > grad_norm_1: tensor(113.4690, device='cuda:0') (tensor(626.2410, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.9046 (0.8559785151631576) + | > loader_time: 0.0154 (0.012564852552593875) + + + --> TIME: 2024-06-25 11:20:40 -- STEP: 343/591 -- GLOBAL_STEP: 31075 + | > loss_disc: 2.591583251953125 (2.6557427356264003) + | > loss_disc_real_0: 0.21631062030792236 (0.19964805356180704) + | > loss_disc_real_1: 0.23458434641361237 (0.22379388762284985) + | > loss_disc_real_2: 0.24380165338516235 (0.23347827118791575) + | > loss_disc_real_3: 0.22792239487171173 (0.2336769732198632) + | > loss_disc_real_4: 0.2542394995689392 (0.22298468536433955) + | > loss_disc_real_5: 0.26448506116867065 (0.23824166091642296) + | > loss_0: 2.591583251953125 (2.6557427356264003) + | > grad_norm_0: tensor(16.8090, device='cuda:0') (tensor(14.9171, device='cuda:0')) + | > loss_gen: 2.005640745162964 (2.0241895483117087) + | > loss_kl: 2.341949939727783 (2.5162931983046897) + | > loss_feat: 4.403738498687744 (4.322221176964893) + | > loss_mel: 20.21177101135254 (20.543301946567734) + | > loss_duration: 1.3074289560317993 (1.3381097542648768) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.270526885986328 (30.74411559591488) + | > grad_norm_1: tensor(1096.2767, device='cuda:0') (tensor(645.7131, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.9277 (0.8612629491455699) + | > loader_time: 0.0174 (0.012835064017737914) + + + --> TIME: 2024-06-25 11:21:04 -- STEP: 368/591 -- GLOBAL_STEP: 31100 + | > loss_disc: 2.606964588165283 (2.65691715867623) + | > loss_disc_real_0: 0.1785927712917328 (0.20019032696828895) + | > loss_disc_real_1: 0.15402132272720337 (0.22355423454681161) + | > loss_disc_real_2: 0.20387235283851624 (0.23354241689262184) + | > loss_disc_real_3: 0.20976628363132477 (0.23381988088721814) + | > loss_disc_real_4: 0.20653487741947174 (0.22262604603462896) + | > loss_disc_real_5: 0.22093060612678528 (0.23829808094255303) + | > loss_0: 2.606964588165283 (2.65691715867623) + | > grad_norm_0: tensor(8.3212, device='cuda:0') (tensor(15.2905, device='cuda:0')) + | > loss_gen: 1.8921985626220703 (2.023988563081494) + | > loss_kl: 2.412952423095703 (2.515078517405882) + | > loss_feat: 4.257997512817383 (4.3128880767718565) + | > loss_mel: 20.279935836791992 (20.538160956424225) + | > loss_duration: 1.351762056350708 (1.3398511707782752) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.194847106933594 (30.729967220969822) + | > grad_norm_1: tensor(748.8785, device='cuda:0') (tensor(657.2830, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.9656 (0.8668346158836198) + | > loader_time: 0.0183 (0.013114387574403178) + + + --> TIME: 2024-06-25 11:21:29 -- STEP: 393/591 -- GLOBAL_STEP: 31125 + | > loss_disc: 2.649545669555664 (2.655435997111197) + | > loss_disc_real_0: 0.17072531580924988 (0.19996595416815227) + | > loss_disc_real_1: 0.2802501320838928 (0.22363427656752455) + | > loss_disc_real_2: 0.22835353016853333 (0.23343095645953074) + | > loss_disc_real_3: 0.2547508478164673 (0.23382298542191357) + | > loss_disc_real_4: 0.23700612783432007 (0.22222074255840168) + | > loss_disc_real_5: 0.25349563360214233 (0.2381615225822871) + | > loss_0: 2.649545669555664 (2.655435997111197) + | > grad_norm_0: tensor(19.7739, device='cuda:0') (tensor(15.4317, device='cuda:0')) + | > loss_gen: 2.1831488609313965 (2.025332793934657) + | > loss_kl: 2.594777822494507 (2.5130547749177192) + | > loss_feat: 4.181832790374756 (4.3074936915293245) + | > loss_mel: 20.91192054748535 (20.540136919676808) + | > loss_duration: 1.3419537544250488 (1.3407139844870148) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.213632583618164 (30.72673210842919) + | > grad_norm_1: tensor(1290.3917, device='cuda:0') (tensor(673.9149, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.9726 (0.8727159821653486) + | > loader_time: 0.0177 (0.013381662562906587) + + + --> TIME: 2024-06-25 11:21:54 -- STEP: 418/591 -- GLOBAL_STEP: 31150 + | > loss_disc: 2.6592562198638916 (2.655709329404329) + | > loss_disc_real_0: 0.13454924523830414 (0.2001822354120501) + | > loss_disc_real_1: 0.1738942712545395 (0.22357911662907115) + | > loss_disc_real_2: 0.25158238410949707 (0.23346944467017525) + | > loss_disc_real_3: 0.22921712696552277 (0.23371161948265642) + | > loss_disc_real_4: 0.1849232017993927 (0.22212734597436554) + | > loss_disc_real_5: 0.21811065077781677 (0.2382158212470666) + | > loss_0: 2.6592562198638916 (2.655709329404329) + | > grad_norm_0: tensor(14.2934, device='cuda:0') (tensor(15.4230, device='cuda:0')) + | > loss_gen: 2.0246617794036865 (2.0245560598715655) + | > loss_kl: 2.5305352210998535 (2.509979649023576) + | > loss_feat: 4.096482753753662 (4.3015324019929375) + | > loss_mel: 19.638429641723633 (20.531148463345033) + | > loss_duration: 1.2856478691101074 (1.3414181388165967) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.57575798034668 (30.70863466856012) + | > grad_norm_1: tensor(194.0452, device='cuda:0') (tensor(677.0027, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 0.9999 (0.878861541383004) + | > loader_time: 0.019 (0.013648150640241262) + + + --> TIME: 2024-06-25 11:22:19 -- STEP: 443/591 -- GLOBAL_STEP: 31175 + | > loss_disc: 2.6536004543304443 (2.6551931286503985) + | > loss_disc_real_0: 0.2611338496208191 (0.20010347543263274) + | > loss_disc_real_1: 0.19210468232631683 (0.2236382161723302) + | > loss_disc_real_2: 0.21658150851726532 (0.23324396348564674) + | > loss_disc_real_3: 0.20537051558494568 (0.2337493500577677) + | > loss_disc_real_4: 0.21102328598499298 (0.22231497600439173) + | > loss_disc_real_5: 0.24031151831150055 (0.2381415762024055) + | > loss_0: 2.6536004543304443 (2.6551931286503985) + | > grad_norm_0: tensor(19.1701, device='cuda:0') (tensor(15.4739, device='cuda:0')) + | > loss_gen: 1.96546471118927 (2.023937170446324) + | > loss_kl: 2.496967315673828 (2.510873639018487) + | > loss_feat: 4.274596214294434 (4.293305731250246) + | > loss_mel: 21.183156967163086 (20.529514114689885) + | > loss_duration: 1.3811228275299072 (1.3428207396384562) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.301307678222656 (30.70045136651928) + | > grad_norm_1: tensor(808.0019, device='cuda:0') (tensor(681.0792, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 1.0497 (0.8859432039626833) + | > loader_time: 0.0162 (0.013918004778653178) + + + --> TIME: 2024-06-25 11:22:46 -- STEP: 468/591 -- GLOBAL_STEP: 31200 + | > loss_disc: 2.644209861755371 (2.6562937065067453) + | > loss_disc_real_0: 0.14512519538402557 (0.2002110558952022) + | > loss_disc_real_1: 0.2370513528585434 (0.2236187796180064) + | > loss_disc_real_2: 0.21664024889469147 (0.2334123943797034) + | > loss_disc_real_3: 0.2044335901737213 (0.2337645208224272) + | > loss_disc_real_4: 0.23002469539642334 (0.2225097974077759) + | > loss_disc_real_5: 0.21925382316112518 (0.2382050297008111) + | > loss_0: 2.644209861755371 (2.6562937065067453) + | > grad_norm_0: tensor(15.1290, device='cuda:0') (tensor(15.4723, device='cuda:0')) + | > loss_gen: 1.944259762763977 (2.021648429397843) + | > loss_kl: 2.480550765991211 (2.508576085424831) + | > loss_feat: 4.205085754394531 (4.282832040236538) + | > loss_mel: 20.648845672607422 (20.511216636396885) + | > loss_duration: 1.3609588146209717 (1.3440270212470982) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.639699935913086 (30.668300172202606) + | > grad_norm_1: tensor(580.1085, device='cuda:0') (tensor(675.4205, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 1.0045 (0.8934928469168832) + | > loader_time: 0.0237 (0.014215974726228629) + + + --> TIME: 2024-06-25 11:23:12 -- STEP: 493/591 -- GLOBAL_STEP: 31225 + | > loss_disc: 2.5901362895965576 (2.6570199280674993) + | > loss_disc_real_0: 0.18619507551193237 (0.20025002082874035) + | > loss_disc_real_1: 0.24066261947155 (0.2234939099204467) + | > loss_disc_real_2: 0.22879597544670105 (0.23342647012058668) + | > loss_disc_real_3: 0.22018757462501526 (0.2337173403337084) + | > loss_disc_real_4: 0.20286211371421814 (0.22278612280835006) + | > loss_disc_real_5: 0.21547524631023407 (0.23826548091417393) + | > loss_0: 2.5901362895965576 (2.6570199280674993) + | > grad_norm_0: tensor(18.2646, device='cuda:0') (tensor(15.4126, device='cuda:0')) + | > loss_gen: 2.150783061981201 (2.0201449222545347) + | > loss_kl: 2.542147636413574 (2.505847517423649) + | > loss_feat: 4.472283363342285 (4.272457174065139) + | > loss_mel: 20.147897720336914 (20.498737493828397) + | > loss_duration: 1.3607711791992188 (1.3453673025173787) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.67388343811035 (30.64255437019147) + | > grad_norm_1: tensor(1137.6052, device='cuda:0') (tensor(672.8864, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 1.026 (0.9009289040284997) + | > loader_time: 0.0194 (0.014514231053133762) + + + --> TIME: 2024-06-25 11:23:41 -- STEP: 518/591 -- GLOBAL_STEP: 31250 + | > loss_disc: 2.663165330886841 (2.657015337446942) + | > loss_disc_real_0: 0.25264236330986023 (0.20019453461911227) + | > loss_disc_real_1: 0.17853447794914246 (0.2235274632177297) + | > loss_disc_real_2: 0.2528093755245209 (0.2335521785046143) + | > loss_disc_real_3: 0.20180615782737732 (0.23375399287027743) + | > loss_disc_real_4: 0.1834566444158554 (0.22254359302143337) + | > loss_disc_real_5: 0.2344384491443634 (0.23838937506942676) + | > loss_0: 2.663165330886841 (2.657015337446942) + | > grad_norm_0: tensor(11.5690, device='cuda:0') (tensor(15.4787, device='cuda:0')) + | > loss_gen: 1.822047472000122 (2.0194657513994043) + | > loss_kl: 2.46148681640625 (2.5045502241053637) + | > loss_feat: 4.0202226638793945 (4.265309083415728) + | > loss_mel: 20.015472412109375 (20.478915328684923) + | > loss_duration: 1.364686131477356 (1.347327171375393) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.683916091918945 (30.615567509271923) + | > grad_norm_1: tensor(46.8641, device='cuda:0') (tensor(679.3021, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 1.1682 (0.9112425895271153) + | > loader_time: 0.0193 (0.014813921166202733) + + + --> TIME: 2024-06-25 11:24:11 -- STEP: 543/591 -- GLOBAL_STEP: 31275 + | > loss_disc: 2.5905632972717285 (2.6576798111474758) + | > loss_disc_real_0: 0.1360539197921753 (0.2002776172663845) + | > loss_disc_real_1: 0.17837192118167877 (0.22349580916581227) + | > loss_disc_real_2: 0.19444473087787628 (0.23345635430698677) + | > loss_disc_real_3: 0.24221399426460266 (0.23376770001968195) + | > loss_disc_real_4: 0.23155874013900757 (0.22270029914971873) + | > loss_disc_real_5: 0.24234521389007568 (0.2384028303030446) + | > loss_0: 2.5905632972717285 (2.6576798111474758) + | > grad_norm_0: tensor(20.5732, device='cuda:0') (tensor(15.6196, device='cuda:0')) + | > loss_gen: 2.029693126678467 (2.0187868404564053) + | > loss_kl: 2.4800121784210205 (2.500823054743955) + | > loss_feat: 4.105751037597656 (4.2543871121731405) + | > loss_mel: 20.504619598388672 (20.457893838759507) + | > loss_duration: 1.4105339050292969 (1.348972023960416) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.530609130859375 (30.580862821575465) + | > grad_norm_1: tensor(869.3305, device='cuda:0') (tensor(682.0737, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 1.3053 (0.924168454129832) + | > loader_time: 0.0242 (0.015158975541481871) + + + --> TIME: 2024-06-25 11:24:42 -- STEP: 568/591 -- GLOBAL_STEP: 31300 + | > loss_disc: 2.7361910343170166 (2.658797275432399) + | > loss_disc_real_0: 0.1771959513425827 (0.20038777576680758) + | > loss_disc_real_1: 0.25563108921051025 (0.22357590108270387) + | > loss_disc_real_2: 0.23759883642196655 (0.23358021702774814) + | > loss_disc_real_3: 0.21421808004379272 (0.23374279368091647) + | > loss_disc_real_4: 0.23872675001621246 (0.222781297449075) + | > loss_disc_real_5: 0.24062679708003998 (0.2384245043856577) + | > loss_0: 2.7361910343170166 (2.658797275432399) + | > grad_norm_0: tensor(12.3476, device='cuda:0') (tensor(15.6402, device='cuda:0')) + | > loss_gen: 1.979114294052124 (2.0168507621741623) + | > loss_kl: 2.3250114917755127 (2.4982903935539893) + | > loss_feat: 3.675795793533325 (4.241771128395908) + | > loss_mel: 19.695175170898438 (20.43717296358566) + | > loss_duration: 1.3768589496612549 (1.350614952369475) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.051956176757812 (30.544700145721436) + | > grad_norm_1: tensor(612.8751, device='cuda:0') (tensor(680.4017, device='cuda:0')) + | > current_lr_0: 0.00019870413513039026 + | > current_lr_1: 0.00019870413513039026 + | > step_time: 1.1916 (0.9366804348750853) + | > loader_time: 0.0304 (0.015569092522204758) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036209441245870387 (+4.34429087537401e-05) + | > avg_loss_disc: 2.756489692850316 (+0.05598557249028602) + | > avg_loss_disc_real_0: 0.4018989509724556 (+0.2023318451135716) + | > avg_loss_disc_real_1: 0.17840656352804063 (-0.0495542769736432) + | > avg_loss_disc_real_2: 0.23941256581468784 (+0.016984520757451965) + | > avg_loss_disc_real_3: 0.21417607557266316 (+0.008983361911266391) + | > avg_loss_disc_real_4: 0.21076384155040093 (-0.03507634173048302) + | > avg_loss_disc_real_5: 0.2246286431525616 (-0.029852822740027152) + | > avg_loss_0: 2.756489692850316 (+0.05598557249028602) + | > avg_loss_gen: 2.0734473517600533 (+0.21185616736716484) + | > avg_loss_kl: 2.539616457959439 (-0.06532619354572589) + | > avg_loss_feat: 4.012117218463976 (+0.04735818315059381) + | > avg_loss_mel: 20.99057948335689 (+0.4460994233476434) + | > avg_loss_duration: 1.3625471896313603 (-0.02401886848693291) + | > avg_loss_1: 30.978307561671482 (+0.6159688259692935) + + + > EPOCH: 54/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 11:25:20)  + + --> TIME: 2024-06-25 11:25:23 -- STEP: 2/591 -- GLOBAL_STEP: 31325 + | > loss_disc: 2.692903518676758 (2.669342517852783) + | > loss_disc_real_0: 0.12882107496261597 (0.12546734511852264) + | > loss_disc_real_1: 0.28034690022468567 (0.25656764954328537) + | > loss_disc_real_2: 0.2127441167831421 (0.23255355656147003) + | > loss_disc_real_3: 0.25820082426071167 (0.22526650875806808) + | > loss_disc_real_4: 0.22921818494796753 (0.22652139514684677) + | > loss_disc_real_5: 0.24862077832221985 (0.2378833070397377) + | > loss_0: 2.692903518676758 (2.669342517852783) + | > grad_norm_0: tensor(11.9727, device='cuda:0') (tensor(15.2452, device='cuda:0')) + | > loss_gen: 2.202889919281006 (2.138232469558716) + | > loss_kl: 2.595665216445923 (2.667852997779846) + | > loss_feat: 4.564251899719238 (4.48579740524292) + | > loss_mel: 21.430055618286133 (21.141926765441895) + | > loss_duration: 1.3252261877059937 (1.3310595750808716) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 32.11809158325195 (31.764870643615723) + | > grad_norm_1: tensor(192.1483, device='cuda:0') (tensor(590.7424, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.7529 (0.7595484256744385) + | > loader_time: 0.0076 (0.008275151252746582) + + + --> TIME: 2024-06-25 11:25:43 -- STEP: 27/591 -- GLOBAL_STEP: 31350 + | > loss_disc: 2.6350021362304688 (2.665220772778546) + | > loss_disc_real_0: 0.2125326693058014 (0.19756749106778038) + | > loss_disc_real_1: 0.1777387261390686 (0.22617662725625215) + | > loss_disc_real_2: 0.21253620088100433 (0.23243654436535305) + | > loss_disc_real_3: 0.22733111679553986 (0.23504937743699109) + | > loss_disc_real_4: 0.2680455148220062 (0.2268381874870371) + | > loss_disc_real_5: 0.252969354391098 (0.23919281032350329) + | > loss_0: 2.6350021362304688 (2.665220772778546) + | > grad_norm_0: tensor(4.7504, device='cuda:0') (tensor(12.3338, device='cuda:0')) + | > loss_gen: 1.8726636171340942 (2.029544035593668) + | > loss_kl: 2.6372365951538086 (2.632619999073169) + | > loss_feat: 4.1507110595703125 (4.45991795151322) + | > loss_mel: 20.523693084716797 (20.62728931285717) + | > loss_duration: 1.3150818347930908 (1.3083217541376746) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.499387741088867 (31.05769305759006) + | > grad_norm_1: tensor(46.4174, device='cuda:0') (tensor(590.9129, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.7911 (0.778500133090549) + | > loader_time: 0.0095 (0.009205676891185618) + + + --> TIME: 2024-06-25 11:26:03 -- STEP: 52/591 -- GLOBAL_STEP: 31375 + | > loss_disc: 2.709260940551758 (2.664081976963925) + | > loss_disc_real_0: 0.19647002220153809 (0.19800143918165794) + | > loss_disc_real_1: 0.25743329524993896 (0.22403848285858446) + | > loss_disc_real_2: 0.2178344428539276 (0.23136440034096056) + | > loss_disc_real_3: 0.25962814688682556 (0.23363989706222826) + | > loss_disc_real_4: 0.2257416993379593 (0.23055939261729902) + | > loss_disc_real_5: 0.23389728367328644 (0.23947582537165055) + | > loss_0: 2.709260940551758 (2.664081976963925) + | > grad_norm_0: tensor(4.0881, device='cuda:0') (tensor(12.5985, device='cuda:0')) + | > loss_gen: 1.9723480939865112 (2.0216334828963642) + | > loss_kl: 2.637211799621582 (2.599548032650581) + | > loss_feat: 4.426312446594238 (4.375995920254635) + | > loss_mel: 20.449174880981445 (20.49423467195951) + | > loss_duration: 1.3195163011550903 (1.3236165986611288) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.804563522338867 (30.815028740809513) + | > grad_norm_1: tensor(160.8228, device='cuda:0') (tensor(525.3545, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.798 (0.7890803263737605) + | > loader_time: 0.0103 (0.009709183986370381) + + + --> TIME: 2024-06-25 11:26:24 -- STEP: 77/591 -- GLOBAL_STEP: 31400 + | > loss_disc: 2.6722123622894287 (2.6689952162953174) + | > loss_disc_real_0: 0.10565448552370071 (0.1973184112023998) + | > loss_disc_real_1: 0.22281841933727264 (0.2246756042752947) + | > loss_disc_real_2: 0.24528630077838898 (0.23185527227915725) + | > loss_disc_real_3: 0.24851644039154053 (0.23323300132503758) + | > loss_disc_real_4: 0.22630047798156738 (0.2313198892327098) + | > loss_disc_real_5: 0.22284674644470215 (0.2395494415775522) + | > loss_0: 2.6722123622894287 (2.6689952162953174) + | > grad_norm_0: tensor(32.7299, device='cuda:0') (tensor(11.8475, device='cuda:0')) + | > loss_gen: 2.1644577980041504 (2.0057203599384845) + | > loss_kl: 2.4064104557037354 (2.572396200972718) + | > loss_feat: 4.300674915313721 (4.323685497432561) + | > loss_mel: 20.709171295166016 (20.508107792247426) + | > loss_duration: 1.3899577856063843 (1.3288122384579149) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.970670700073242 (30.73872204570027) + | > grad_norm_1: tensor(740.7598, device='cuda:0') (tensor(463.8426, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.8265 (0.7987137831650771) + | > loader_time: 0.0105 (0.00994074499452269) + + + --> TIME: 2024-06-25 11:26:45 -- STEP: 102/591 -- GLOBAL_STEP: 31425 + | > loss_disc: 2.630465030670166 (2.675030542354958) + | > loss_disc_real_0: 0.174851655960083 (0.19952010319513433) + | > loss_disc_real_1: 0.2382836937904358 (0.22430810273862353) + | > loss_disc_real_2: 0.2005147486925125 (0.23231523279465882) + | > loss_disc_real_3: 0.2075304090976715 (0.23310130995278264) + | > loss_disc_real_4: 0.22377948462963104 (0.23208556996256696) + | > loss_disc_real_5: 0.23406019806861877 (0.23961439173595578) + | > loss_0: 2.630465030670166 (2.675030542354958) + | > grad_norm_0: tensor(13.6667, device='cuda:0') (tensor(12.4699, device='cuda:0')) + | > loss_gen: 2.118741989135742 (1.9986123010223982) + | > loss_kl: 2.449406147003174 (2.5620316407259773) + | > loss_feat: 4.539266586303711 (4.302272228633655) + | > loss_mel: 20.512897491455078 (20.51358161253088) + | > loss_duration: 1.3450570106506348 (1.3357504571185392) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.965368270874023 (30.71224816640218) + | > grad_norm_1: tensor(1004.3995, device='cuda:0') (tensor(471.2161, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.8231 (0.8064143447314992) + | > loader_time: 0.0108 (0.010188946536943022) + + + --> TIME: 2024-06-25 11:27:06 -- STEP: 127/591 -- GLOBAL_STEP: 31450 + | > loss_disc: 2.77420711517334 (2.673759723272849) + | > loss_disc_real_0: 0.10235955566167831 (0.1994092125944265) + | > loss_disc_real_1: 0.2195972055196762 (0.22400985810700363) + | > loss_disc_real_2: 0.23914872109889984 (0.2334159010742593) + | > loss_disc_real_3: 0.23398089408874512 (0.23347898097488823) + | > loss_disc_real_4: 0.22277945280075073 (0.230328561283472) + | > loss_disc_real_5: 0.24462246894836426 (0.2399428399998372) + | > loss_0: 2.77420711517334 (2.673759723272849) + | > grad_norm_0: tensor(33.6846, device='cuda:0') (tensor(13.0287, device='cuda:0')) + | > loss_gen: 2.2258074283599854 (2.002176616135544) + | > loss_kl: 2.5184829235076904 (2.5482546438382365) + | > loss_feat: 4.131742000579834 (4.297087870244903) + | > loss_mel: 20.078582763671875 (20.513607265442378) + | > loss_duration: 1.3468561172485352 (1.3374539341513567) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.301471710205078 (30.698580343892257) + | > grad_norm_1: tensor(985.6705, device='cuda:0') (tensor(526.4828, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.8395 (0.813277749564704) + | > loader_time: 0.0108 (0.010397852875116301) + + + --> TIME: 2024-06-25 11:27:28 -- STEP: 152/591 -- GLOBAL_STEP: 31475 + | > loss_disc: 2.713103771209717 (2.671222250712545) + | > loss_disc_real_0: 0.19819428026676178 (0.1998801855370402) + | > loss_disc_real_1: 0.2026468962430954 (0.22454168598510718) + | > loss_disc_real_2: 0.22303226590156555 (0.23297083662136606) + | > loss_disc_real_3: 0.2456972897052765 (0.23341814350140722) + | > loss_disc_real_4: 0.20063991844654083 (0.22813471514535577) + | > loss_disc_real_5: 0.22478096187114716 (0.23982505098377405) + | > loss_0: 2.713103771209717 (2.671222250712545) + | > grad_norm_0: tensor(16.5204, device='cuda:0') (tensor(13.6745, device='cuda:0')) + | > loss_gen: 2.047084331512451 (2.008195153976741) + | > loss_kl: 2.376323938369751 (2.5387701674511556) + | > loss_feat: 4.284322261810303 (4.301384168235878) + | > loss_mel: 20.759815216064453 (20.49101638793946) + | > loss_duration: 1.333809733390808 (1.3360956413181206) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.801355361938477 (30.675461555782118) + | > grad_norm_1: tensor(1174.5306, device='cuda:0') (tensor(580.4575, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.8529 (0.8196721939664139) + | > loader_time: 0.0115 (0.010630720540096885) + + + --> TIME: 2024-06-25 11:27:50 -- STEP: 177/591 -- GLOBAL_STEP: 31500 + | > loss_disc: 2.5863523483276367 (2.669212955539509) + | > loss_disc_real_0: 0.23857057094573975 (0.20041972116730308) + | > loss_disc_real_1: 0.14709794521331787 (0.22414866325545446) + | > loss_disc_real_2: 0.23153050243854523 (0.23250912432953463) + | > loss_disc_real_3: 0.2147965282201767 (0.23336767805161449) + | > loss_disc_real_4: 0.21647317707538605 (0.22699875616084383) + | > loss_disc_real_5: 0.23371416330337524 (0.23990448139144877) + | > loss_0: 2.5863523483276367 (2.669212955539509) + | > grad_norm_0: tensor(18.2158, device='cuda:0') (tensor(13.9851, device='cuda:0')) + | > loss_gen: 2.157876968383789 (2.009976887433542) + | > loss_kl: 2.633730173110962 (2.5341593963278206) + | > loss_feat: 4.5025482177734375 (4.303053843773018) + | > loss_mel: 19.995710372924805 (20.47209344206557) + | > loss_duration: 1.3405663967132568 (1.3347613824962903) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.63043212890625 (30.654044986444678) + | > grad_norm_1: tensor(1003.8360, device='cuda:0') (tensor(603.1477, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.8669 (0.825905150612869) + | > loader_time: 0.0116 (0.010898208887563588) + + + --> TIME: 2024-06-25 11:28:12 -- STEP: 202/591 -- GLOBAL_STEP: 31525 + | > loss_disc: 2.584662914276123 (2.66455619169934) + | > loss_disc_real_0: 0.14056961238384247 (0.19975319316629137) + | > loss_disc_real_1: 0.20369833707809448 (0.22415977517271987) + | > loss_disc_real_2: 0.23795315623283386 (0.23307979453613262) + | > loss_disc_real_3: 0.22183577716350555 (0.2337019330646732) + | > loss_disc_real_4: 0.18908052146434784 (0.2249530541277168) + | > loss_disc_real_5: 0.2425723373889923 (0.2397742227162465) + | > loss_0: 2.584662914276123 (2.66455619169934) + | > grad_norm_0: tensor(15.7302, device='cuda:0') (tensor(14.2962, device='cuda:0')) + | > loss_gen: 2.0561842918395996 (2.01993338601424) + | > loss_kl: 2.360758066177368 (2.52957814046652) + | > loss_feat: 4.4801177978515625 (4.322216462380819) + | > loss_mel: 20.463340759277344 (20.477940984291614) + | > loss_duration: 1.2992961406707764 (1.3343777444102976) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.659696578979492 (30.68404672641565) + | > grad_norm_1: tensor(674.9029, device='cuda:0') (tensor(640.2944, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.9073 (0.8316960747879332) + | > loader_time: 0.0144 (0.01110694668080547) + + + --> TIME: 2024-06-25 11:28:35 -- STEP: 227/591 -- GLOBAL_STEP: 31550 + | > loss_disc: 2.681431770324707 (2.6665693295684685) + | > loss_disc_real_0: 0.16173699498176575 (0.20062846618208066) + | > loss_disc_real_1: 0.21704496443271637 (0.22421869077608975) + | > loss_disc_real_2: 0.22568370401859283 (0.23285924383984788) + | > loss_disc_real_3: 0.2640210688114166 (0.23341654510224968) + | > loss_disc_real_4: 0.21297979354858398 (0.22583190834207156) + | > loss_disc_real_5: 0.2191021889448166 (0.239417788675178) + | > loss_0: 2.681431770324707 (2.6665693295684685) + | > grad_norm_0: tensor(11.6407, device='cuda:0') (tensor(14.3585, device='cuda:0')) + | > loss_gen: 1.9322589635849 (2.0164087947769826) + | > loss_kl: 2.6466355323791504 (2.5280533088986568) + | > loss_feat: 4.310126781463623 (4.31086348016882) + | > loss_mel: 20.17919158935547 (20.48154793121742) + | > loss_duration: 1.343264102935791 (1.3339667230975785) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.411476135253906 (30.670840217153405) + | > grad_norm_1: tensor(75.1026, device='cuda:0') (tensor(617.7137, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.8682 (0.8365945868555146) + | > loader_time: 0.0151 (0.01143447102954209) + + + --> TIME: 2024-06-25 11:28:58 -- STEP: 252/591 -- GLOBAL_STEP: 31575 + | > loss_disc: 2.668163299560547 (2.666465272033026) + | > loss_disc_real_0: 0.17055001854896545 (0.2002235921543269) + | > loss_disc_real_1: 0.27429038286209106 (0.22423768711705055) + | > loss_disc_real_2: 0.2541782259941101 (0.23301782991204942) + | > loss_disc_real_3: 0.2194729447364807 (0.23322897711916576) + | > loss_disc_real_4: 0.23886869847774506 (0.22675821664077894) + | > loss_disc_real_5: 0.23883767426013947 (0.2392996053492266) + | > loss_0: 2.668163299560547 (2.666465272033026) + | > grad_norm_0: tensor(8.4087, device='cuda:0') (tensor(13.8496, device='cuda:0')) + | > loss_gen: 2.0332887172698975 (2.012422625980679) + | > loss_kl: 2.512850761413574 (2.5263655450608997) + | > loss_feat: 4.041932582855225 (4.301806740344521) + | > loss_mel: 19.938339233398438 (20.465862523941773) + | > loss_duration: 1.3205146789550781 (1.3344469377918848) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.846925735473633 (30.640904366023957) + | > grad_norm_1: tensor(523.8697, device='cuda:0') (tensor(581.8804, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.8828 (0.8420853245825995) + | > loader_time: 0.015 (0.011770867166065034) + + + --> TIME: 2024-06-25 11:29:21 -- STEP: 277/591 -- GLOBAL_STEP: 31600 + | > loss_disc: 2.771113872528076 (2.6671983687886267) + | > loss_disc_real_0: 0.3512122333049774 (0.20083492948582884) + | > loss_disc_real_1: 0.21000294387340546 (0.22394466755192202) + | > loss_disc_real_2: 0.2188856303691864 (0.23292823695318793) + | > loss_disc_real_3: 0.2243146151304245 (0.2333012146317141) + | > loss_disc_real_4: 0.22602906823158264 (0.2269793153347091) + | > loss_disc_real_5: 0.2366320788860321 (0.2391848347354882) + | > loss_0: 2.771113872528076 (2.6671983687886267) + | > grad_norm_0: tensor(25.8684, device='cuda:0') (tensor(13.6675, device='cuda:0')) + | > loss_gen: 1.8278045654296875 (2.0098824178268755) + | > loss_kl: 2.5570106506347656 (2.5244088775414424) + | > loss_feat: 4.002310752868652 (4.296076666146843) + | > loss_mel: 19.66416358947754 (20.464068374909225) + | > loss_duration: 1.3423205614089966 (1.3347183874798165) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.39361000061035 (30.629154742409607) + | > grad_norm_1: tensor(135.1142, device='cuda:0') (tensor(564.4387, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.937 (0.8476530033758832) + | > loader_time: 0.0154 (0.012064573997194586) + + + --> TIME: 2024-06-25 11:29:44 -- STEP: 302/591 -- GLOBAL_STEP: 31625 + | > loss_disc: 2.6908891201019287 (2.667235338924738) + | > loss_disc_real_0: 0.21964703500270844 (0.20080990583592687) + | > loss_disc_real_1: 0.17911270260810852 (0.2242991548796363) + | > loss_disc_real_2: 0.24774537980556488 (0.23322575490008915) + | > loss_disc_real_3: 0.2548951506614685 (0.23354855826161555) + | > loss_disc_real_4: 0.19655653834342957 (0.22575714409548714) + | > loss_disc_real_5: 0.2597961127758026 (0.23914986399822677) + | > loss_0: 2.6908891201019287 (2.667235338924738) + | > grad_norm_0: tensor(11.3048, device='cuda:0') (tensor(14.3405, device='cuda:0')) + | > loss_gen: 1.9929596185684204 (2.0142921276439885) + | > loss_kl: 2.596083879470825 (2.520410529035606) + | > loss_feat: 4.191348552703857 (4.295947734093827) + | > loss_mel: 20.785160064697266 (20.457338484707268) + | > loss_duration: 1.3088587522506714 (1.335304244069863) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.874412536621094 (30.62329313770825) + | > grad_norm_1: tensor(358.0831, device='cuda:0') (tensor(598.9461, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.9755 (0.853577478831967) + | > loader_time: 0.0135 (0.012347956366886367) + + + --> TIME: 2024-06-25 11:30:08 -- STEP: 327/591 -- GLOBAL_STEP: 31650 + | > loss_disc: 2.6240499019622803 (2.664863568927171) + | > loss_disc_real_0: 0.14100047945976257 (0.20027787324941113) + | > loss_disc_real_1: 0.18646088242530823 (0.22452523356548507) + | > loss_disc_real_2: 0.22999604046344757 (0.2332076153259394) + | > loss_disc_real_3: 0.2301895022392273 (0.2335087748113393) + | > loss_disc_real_4: 0.188431054353714 (0.22475704960866805) + | > loss_disc_real_5: 0.22210824489593506 (0.2390404908208672) + | > loss_0: 2.6240499019622803 (2.664863568927171) + | > grad_norm_0: tensor(13.1767, device='cuda:0') (tensor(14.5165, device='cuda:0')) + | > loss_gen: 1.991231918334961 (2.0162483515724845) + | > loss_kl: 2.597177505493164 (2.5189380842611335) + | > loss_feat: 4.088597774505615 (4.297909823397251) + | > loss_mel: 19.5803279876709 (20.45125712823431) + | > loss_duration: 1.3708934783935547 (1.3362726509024243) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 29.62822914123535 (30.62062602961829) + | > grad_norm_1: tensor(697.0940, device='cuda:0') (tensor(621.9907, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.9186 (0.858722333149808) + | > loader_time: 0.0142 (0.012594400195900452) + + + --> TIME: 2024-06-25 11:30:32 -- STEP: 352/591 -- GLOBAL_STEP: 31675 + | > loss_disc: 2.6229920387268066 (2.663163126192311) + | > loss_disc_real_0: 0.19434407353401184 (0.20010438358241853) + | > loss_disc_real_1: 0.17677123844623566 (0.22421343209729952) + | > loss_disc_real_2: 0.23833855986595154 (0.23329519692131065) + | > loss_disc_real_3: 0.18080677092075348 (0.23357357228682799) + | > loss_disc_real_4: 0.16518747806549072 (0.2242774784310975) + | > loss_disc_real_5: 0.2508055865764618 (0.23913158434019846) + | > loss_0: 2.6229920387268066 (2.663163126192311) + | > grad_norm_0: tensor(23.8981, device='cuda:0') (tensor(15.2019, device='cuda:0')) + | > loss_gen: 2.0469000339508057 (2.023653666742822) + | > loss_kl: 2.523085832595825 (2.515453060242265) + | > loss_feat: 4.410630226135254 (4.306797114962884) + | > loss_mel: 20.950712203979492 (20.452970873225834) + | > loss_duration: 1.3656891584396362 (1.337542155588215) + | > amp_scaler: 64.0 (125.45454545454545) + | > loss_1: 31.297016143798828 (30.636416852474213) + | > grad_norm_1: tensor(1183.1797, device='cuda:0') (tensor(647.9086, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.9324 (0.8644860515540297) + | > loader_time: 0.0175 (0.012837914580648596) + + + --> TIME: 2024-06-25 11:30:57 -- STEP: 377/591 -- GLOBAL_STEP: 31700 + | > loss_disc: 2.7739322185516357 (2.6642460184324968) + | > loss_disc_real_0: 0.2712398171424866 (0.2003434145877468) + | > loss_disc_real_1: 0.2670404314994812 (0.22485652716153495) + | > loss_disc_real_2: 0.23664043843746185 (0.23343654040158268) + | > loss_disc_real_3: 0.2369781732559204 (0.23367015906765226) + | > loss_disc_real_4: 0.18123234808444977 (0.22393224856739652) + | > loss_disc_real_5: 0.22132891416549683 (0.23904530190188308) + | > loss_0: 2.7739322185516357 (2.6642460184324968) + | > grad_norm_0: tensor(11.6738, device='cuda:0') (tensor(15.3302, device='cuda:0')) + | > loss_gen: 1.777583360671997 (2.0230491585061157) + | > loss_kl: 2.5393636226654053 (2.514394725349286) + | > loss_feat: 4.057596683502197 (4.29882723899356) + | > loss_mel: 20.660362243652344 (20.444829859847424) + | > loss_duration: 1.3761723041534424 (1.3391004661666306) + | > amp_scaler: 64.0 (121.37931034482759) + | > loss_1: 30.41107940673828 (30.620201454870898) + | > grad_norm_1: tensor(81.4067, device='cuda:0') (tensor(655.1491, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 1.0979 (0.8723812779950211) + | > loader_time: 0.0151 (0.013070427138229897) + + + --> TIME: 2024-06-25 11:31:22 -- STEP: 402/591 -- GLOBAL_STEP: 31725 + | > loss_disc: 2.6580750942230225 (2.6653720988563054) + | > loss_disc_real_0: 0.18878188729286194 (0.20008743811973284) + | > loss_disc_real_1: 0.2519231140613556 (0.22493635470149528) + | > loss_disc_real_2: 0.23774145543575287 (0.23339468585466272) + | > loss_disc_real_3: 0.23106816411018372 (0.2337653536108596) + | > loss_disc_real_4: 0.23933234810829163 (0.22437807813805727) + | > loss_disc_real_5: 0.244320809841156 (0.23910557011614986) + | > loss_0: 2.6580750942230225 (2.6653720988563054) + | > grad_norm_0: tensor(11.5700, device='cuda:0') (tensor(15.2286, device='cuda:0')) + | > loss_gen: 1.9320042133331299 (2.0187385227549735) + | > loss_kl: 2.5107638835906982 (2.51293784231689) + | > loss_feat: 3.827467441558838 (4.281923400821975) + | > loss_mel: 20.413930892944336 (20.444460759708537) + | > loss_duration: 1.4009590148925781 (1.340322687258175) + | > amp_scaler: 64.0 (117.81094527363184) + | > loss_1: 30.085126876831055 (30.59838321078476) + | > grad_norm_1: tensor(293.1799, device='cuda:0') (tensor(639.8897, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.9881 (0.8797630273287569) + | > loader_time: 0.0192 (0.013343674626516465) + + + --> TIME: 2024-06-25 11:31:48 -- STEP: 427/591 -- GLOBAL_STEP: 31750 + | > loss_disc: 2.690807342529297 (2.665616549150169) + | > loss_disc_real_0: 0.25010156631469727 (0.20015962859855596) + | > loss_disc_real_1: 0.20302070677280426 (0.22464428761803854) + | > loss_disc_real_2: 0.25440001487731934 (0.2333194633348206) + | > loss_disc_real_3: 0.2745072543621063 (0.23376430148821525) + | > loss_disc_real_4: 0.24534112215042114 (0.22500972910424305) + | > loss_disc_real_5: 0.2644086480140686 (0.23912307229198393) + | > loss_0: 2.690807342529297 (2.665616549150169) + | > grad_norm_0: tensor(12.8394, device='cuda:0') (tensor(14.9340, device='cuda:0')) + | > loss_gen: 1.8465397357940674 (2.0163867172089343) + | > loss_kl: 2.546218156814575 (2.5105081933443687) + | > loss_feat: 4.0897440910339355 (4.272518452213297) + | > loss_mel: 20.595956802368164 (20.433430477383546) + | > loss_duration: 1.3733177185058594 (1.3413460528264283) + | > amp_scaler: 64.0 (114.66042154566745) + | > loss_1: 30.4517765045166 (30.574189885159566) + | > grad_norm_1: tensor(459.6932, device='cuda:0') (tensor(622.1031, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.9756 (0.886310737082774) + | > loader_time: 0.0169 (0.013613686349408688) + + + --> TIME: 2024-06-25 11:32:13 -- STEP: 452/591 -- GLOBAL_STEP: 31775 + | > loss_disc: 2.7149481773376465 (2.6663791600581828) + | > loss_disc_real_0: 0.17371928691864014 (0.20022214472755387) + | > loss_disc_real_1: 0.2794501781463623 (0.22465422307759259) + | > loss_disc_real_2: 0.243019238114357 (0.23330525160495158) + | > loss_disc_real_3: 0.22318816184997559 (0.23386618425756428) + | > loss_disc_real_4: 0.23991607129573822 (0.22510814643671026) + | > loss_disc_real_5: 0.2797446548938751 (0.23901784176056365) + | > loss_0: 2.7149481773376465 (2.6663791600581828) + | > grad_norm_0: tensor(19.4941, device='cuda:0') (tensor(15.0312, device='cuda:0')) + | > loss_gen: 2.0735654830932617 (2.014472093202374) + | > loss_kl: 2.557748317718506 (2.5083641219983077) + | > loss_feat: 4.02872896194458 (4.262494458033976) + | > loss_mel: 20.046369552612305 (20.42120655026056) + | > loss_duration: 1.3385158777236938 (1.3423453208092042) + | > amp_scaler: 64.0 (111.85840707964601) + | > loss_1: 30.04492950439453 (30.548882547733005) + | > grad_norm_1: tensor(884.8884, device='cuda:0') (tensor(618.1841, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 0.9976 (0.8932412018818138) + | > loader_time: 0.0212 (0.013886544029269598) + + + --> TIME: 2024-06-25 11:32:39 -- STEP: 477/591 -- GLOBAL_STEP: 31800 + | > loss_disc: 2.6494193077087402 (2.666088488116955) + | > loss_disc_real_0: 0.17174319922924042 (0.20008148815271984) + | > loss_disc_real_1: 0.22403785586357117 (0.22484400131167606) + | > loss_disc_real_2: 0.2315567582845688 (0.23330309702790258) + | > loss_disc_real_3: 0.2207220196723938 (0.23390907470910055) + | > loss_disc_real_4: 0.238648921251297 (0.22490767669127923) + | > loss_disc_real_5: 0.2707855999469757 (0.23909537787832305) + | > loss_0: 2.6494193077087402 (2.666088488116955) + | > grad_norm_0: tensor(7.2514, device='cuda:0') (tensor(15.1512, device='cuda:0')) + | > loss_gen: 2.0059711933135986 (2.0145821291445714) + | > loss_kl: 2.460437297821045 (2.5043148264695034) + | > loss_feat: 3.9474503993988037 (4.254730587485452) + | > loss_mel: 20.099393844604492 (20.41883156334579) + | > loss_duration: 1.3613831996917725 (1.3436159265366243) + | > amp_scaler: 64.0 (109.35010482180293) + | > loss_1: 29.874637603759766 (30.53607502023629) + | > grad_norm_1: tensor(490.7693, device='cuda:0') (tensor(628.3217, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 1.0186 (0.8997363284448907) + | > loader_time: 0.0177 (0.014178837870151992) + + + --> TIME: 2024-06-25 11:33:06 -- STEP: 502/591 -- GLOBAL_STEP: 31825 + | > loss_disc: 2.7205450534820557 (2.664982257611249) + | > loss_disc_real_0: 0.14937961101531982 (0.2000985568114723) + | > loss_disc_real_1: 0.19146056473255157 (0.22471194197813354) + | > loss_disc_real_2: 0.23482611775398254 (0.23331381849558705) + | > loss_disc_real_3: 0.186478391289711 (0.23387513683730388) + | > loss_disc_real_4: 0.20564351975917816 (0.2241982848223937) + | > loss_disc_real_5: 0.24504871666431427 (0.2390249216877132) + | > loss_0: 2.7205450534820557 (2.664982257611249) + | > grad_norm_0: tensor(14.9399, device='cuda:0') (tensor(15.4851, device='cuda:0')) + | > loss_gen: 1.999165415763855 (2.0169599773399405) + | > loss_kl: 2.3896660804748535 (2.501719289566893) + | > loss_feat: 3.773300886154175 (4.253380493338842) + | > loss_mel: 19.748403549194336 (20.40654364810047) + | > loss_duration: 1.3887559175491333 (1.3450834181204263) + | > amp_scaler: 64.0 (107.09163346613546) + | > loss_1: 29.29928970336914 (30.523686815543005) + | > grad_norm_1: tensor(61.4964, device='cuda:0') (tensor(642.8134, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 1.0828 (0.9076020746117094) + | > loader_time: 0.0208 (0.014470084729897548) + + + --> TIME: 2024-06-25 11:33:35 -- STEP: 527/591 -- GLOBAL_STEP: 31850 + | > loss_disc: 2.527218818664551 (2.665134226800822) + | > loss_disc_real_0: 0.2277430295944214 (0.20021103241920918) + | > loss_disc_real_1: 0.20501220226287842 (0.22470230390043816) + | > loss_disc_real_2: 0.23826028406620026 (0.2333176450722584) + | > loss_disc_real_3: 0.18632255494594574 (0.23394788948702405) + | > loss_disc_real_4: 0.1654995083808899 (0.2242975467409536) + | > loss_disc_real_5: 0.20846296846866608 (0.23896948894241957) + | > loss_0: 2.527218818664551 (2.665134226800822) + | > grad_norm_0: tensor(22.0653, device='cuda:0') (tensor(15.6243, device='cuda:0')) + | > loss_gen: 2.117624521255493 (2.0160006877367156) + | > loss_kl: 2.438326835632324 (2.5007001050735576) + | > loss_feat: 4.595037460327148 (4.245809466834316) + | > loss_mel: 20.247831344604492 (20.399621952643205) + | > loss_duration: 1.3380850553512573 (1.3468409989318995) + | > amp_scaler: 64.0 (105.04743833017078) + | > loss_1: 30.736906051635742 (30.50897320307636) + | > grad_norm_1: tensor(903.3814, device='cuda:0') (tensor(643.4949, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 1.1801 (0.9175720486288053) + | > loader_time: 0.0196 (0.014802708797708182) + + + --> TIME: 2024-06-25 11:34:05 -- STEP: 552/591 -- GLOBAL_STEP: 31875 + | > loss_disc: 2.571941375732422 (2.6642024745975714) + | > loss_disc_real_0: 0.13925713300704956 (0.20000234333987255) + | > loss_disc_real_1: 0.24419492483139038 (0.22479160726610298) + | > loss_disc_real_2: 0.24249579012393951 (0.2332653808744921) + | > loss_disc_real_3: 0.2093714475631714 (0.23401901530830757) + | > loss_disc_real_4: 0.18601080775260925 (0.2237625232749228) + | > loss_disc_real_5: 0.23203745484352112 (0.23898434328536192) + | > loss_0: 2.571941375732422 (2.6642024745975714) + | > grad_norm_0: tensor(21.8892, device='cuda:0') (tensor(15.8614, device='cuda:0')) + | > loss_gen: 2.1097147464752197 (2.018096885603408) + | > loss_kl: 2.4679818153381348 (2.4986693561077136) + | > loss_feat: 4.031096935272217 (4.243940546892693) + | > loss_mel: 19.83551025390625 (20.396514661070228) + | > loss_duration: 1.4319939613342285 (1.3487374383038375) + | > amp_scaler: 64.0 (103.18840579710144) + | > loss_1: 29.876296997070312 (30.505958861198977) + | > grad_norm_1: tensor(1173.0266, device='cuda:0') (tensor(652.2324, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 1.1063 (0.9283570487430133) + | > loader_time: 0.0205 (0.015148447043653847) + + + --> TIME: 2024-06-25 11:34:36 -- STEP: 577/591 -- GLOBAL_STEP: 31900 + | > loss_disc: 2.6643290519714355 (2.6637232278827154) + | > loss_disc_real_0: 0.17928221821784973 (0.20013973518457126) + | > loss_disc_real_1: 0.24866220355033875 (0.22487378626596993) + | > loss_disc_real_2: 0.23805424571037292 (0.23313671600983915) + | > loss_disc_real_3: 0.21623961627483368 (0.23401689085224886) + | > loss_disc_real_4: 0.22122876346111298 (0.22344953382139193) + | > loss_disc_real_5: 0.2455555945634842 (0.23905189164899665) + | > loss_0: 2.6643290519714355 (2.6637232278827154) + | > grad_norm_0: tensor(9.4295, device='cuda:0') (tensor(16.2655, device='cuda:0')) + | > loss_gen: 1.9138649702072144 (2.0198631745277296) + | > loss_kl: 2.4530868530273438 (2.4961681250269754) + | > loss_feat: 3.8131701946258545 (4.239168648083456) + | > loss_mel: 20.08184814453125 (20.386657982599797) + | > loss_duration: 1.4071409702301025 (1.3504253549427059) + | > amp_scaler: 64.0 (101.49046793760832) + | > loss_1: 29.669111251831055 (30.492283260801635) + | > grad_norm_1: tensor(476.9928, device='cuda:0') (tensor(661.9311, device='cuda:0')) + | > current_lr_0: 0.00019867929711349895 + | > current_lr_1: 0.00019867929711349895 + | > step_time: 1.2326 (0.9410185309795203) + | > loader_time: 0.0218 (0.01554263733082049) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003591882421615276 (-2.9061702971762762e-05) + | > avg_loss_disc: 2.7028872358038076 (-0.053602457046508345) + | > avg_loss_disc_real_0: 0.2948609289336713 (-0.10703802203878432) + | > avg_loss_disc_real_1: 0.2211575704686185 (+0.042751006940577874) + | > avg_loss_disc_real_2: 0.2334447648931057 (-0.005967800921582139) + | > avg_loss_disc_real_3: 0.23258168805152812 (+0.018405612478864963) + | > avg_loss_disc_real_4: 0.2886586122690364 (+0.07789477071863546) + | > avg_loss_disc_real_5: 0.24647969958630014 (+0.021851056433738553) + | > avg_loss_0: 2.7028872358038076 (-0.053602457046508345) + | > avg_loss_gen: 2.1358634182747376 (+0.06241606651468423) + | > avg_loss_kl: 2.5316744560891022 (-0.007942001870336668) + | > avg_loss_feat: 4.02943978411086 (+0.01732256564688406) + | > avg_loss_mel: 20.299466234572392 (-0.6911132487844966) + | > avg_loss_duration: 1.3711286580308955 (+0.008581468399535108) + | > avg_loss_1: 30.367572784423828 (-0.6107347772476537) + + + > EPOCH: 55/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 11:35:01)  + + --> TIME: 2024-06-25 11:35:11 -- STEP: 11/591 -- GLOBAL_STEP: 31925 + | > loss_disc: 2.585653305053711 (2.654698220166293) + | > loss_disc_real_0: 0.19728584587574005 (0.20267508111216806) + | > loss_disc_real_1: 0.21056441962718964 (0.22575709359212356) + | > loss_disc_real_2: 0.20709499716758728 (0.22951518134637314) + | > loss_disc_real_3: 0.2226439118385315 (0.23499445075338538) + | > loss_disc_real_4: 0.23270530998706818 (0.22868971933018079) + | > loss_disc_real_5: 0.23578499257564545 (0.2413062263618816) + | > loss_0: 2.585653305053711 (2.654698220166293) + | > grad_norm_0: tensor(11.8326, device='cuda:0') (tensor(16.4504, device='cuda:0')) + | > loss_gen: 2.0789284706115723 (2.0261580727317114) + | > loss_kl: 2.58725643157959 (2.602673205462369) + | > loss_feat: 4.3716044425964355 (4.479730866172097) + | > loss_mel: 20.690746307373047 (20.506299105557527) + | > loss_duration: 1.2816612720489502 (1.3158930756829001) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.010196685791016 (30.930754921653055) + | > grad_norm_1: tensor(701.9264, device='cuda:0') (tensor(702.1733, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.7903 (0.7695499766956676) + | > loader_time: 0.0095 (0.00836524096402255) + + + --> TIME: 2024-06-25 11:35:31 -- STEP: 36/591 -- GLOBAL_STEP: 31950 + | > loss_disc: 2.636200428009033 (2.652276078859965) + | > loss_disc_real_0: 0.2064821720123291 (0.20162418194943005) + | > loss_disc_real_1: 0.2724441587924957 (0.22303312520186105) + | > loss_disc_real_2: 0.2368064969778061 (0.23408084528313744) + | > loss_disc_real_3: 0.21565483510494232 (0.23581456806924608) + | > loss_disc_real_4: 0.19851064682006836 (0.21850773526562584) + | > loss_disc_real_5: 0.19640928506851196 (0.23824572770131958) + | > loss_0: 2.636200428009033 (2.652276078859965) + | > grad_norm_0: tensor(20.8249, device='cuda:0') (tensor(18.4714, device='cuda:0')) + | > loss_gen: 2.184765100479126 (2.0412459174791966) + | > loss_kl: 2.5796263217926025 (2.600709637006124) + | > loss_feat: 4.58840274810791 (4.473584466510349) + | > loss_mel: 21.000011444091797 (20.539147059122726) + | > loss_duration: 1.347682237625122 (1.3169562021891277) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.700489044189453 (30.971643500857883) + | > grad_norm_1: tensor(1242.3950, device='cuda:0') (tensor(861.4232, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.8059 (0.783106287320455) + | > loader_time: 0.0105 (0.00911978218290541) + + + --> TIME: 2024-06-25 11:35:52 -- STEP: 61/591 -- GLOBAL_STEP: 31975 + | > loss_disc: 2.589974880218506 (2.6387311005201495) + | > loss_disc_real_0: 0.23004034161567688 (0.20335737823462877) + | > loss_disc_real_1: 0.23640061914920807 (0.22363324658792527) + | > loss_disc_real_2: 0.2349097579717636 (0.23316694429663362) + | > loss_disc_real_3: 0.26182636618614197 (0.23502342305222496) + | > loss_disc_real_4: 0.2131793200969696 (0.21210378970279067) + | > loss_disc_real_5: 0.2594694495201111 (0.23900711634120003) + | > loss_0: 2.589974880218506 (2.6387311005201495) + | > grad_norm_0: tensor(15.9947, device='cuda:0') (tensor(22.0840, device='cuda:0')) + | > loss_gen: 1.9874536991119385 (2.085333577922132) + | > loss_kl: 2.5928006172180176 (2.5594907784071124) + | > loss_feat: 4.2935943603515625 (4.509507585744389) + | > loss_mel: 20.169198989868164 (20.57719233778657) + | > loss_duration: 1.3384674787521362 (1.3249975872821493) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.381513595581055 (31.05652190036461) + | > grad_norm_1: tensor(1082.5703, device='cuda:0') (tensor(1018.9009, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.8142 (0.792837170303845) + | > loader_time: 0.0102 (0.009403807217957543) + + + --> TIME: 2024-06-25 11:36:13 -- STEP: 86/591 -- GLOBAL_STEP: 32000 + | > loss_disc: 2.5963897705078125 (2.6346076227897823) + | > loss_disc_real_0: 0.1548013836145401 (0.2002218698346338) + | > loss_disc_real_1: 0.20710664987564087 (0.22309502959251404) + | > loss_disc_real_2: 0.23948469758033752 (0.23274139280236045) + | > loss_disc_real_3: 0.2424858957529068 (0.2339547775512518) + | > loss_disc_real_4: 0.19419334828853607 (0.21169560870458914) + | > loss_disc_real_5: 0.21026812493801117 (0.23755755136872447) + | > loss_0: 2.5963897705078125 (2.6346076227897823) + | > grad_norm_0: tensor(27.3102, device='cuda:0') (tensor(21.5586, device='cuda:0')) + | > loss_gen: 2.196397066116333 (2.081684058488801) + | > loss_kl: 2.3993921279907227 (2.5470824657484537) + | > loss_feat: 4.868781089782715 (4.500247018281803) + | > loss_mel: 20.72066879272461 (20.614163664884355) + | > loss_duration: 1.3683046102523804 (1.3327123356419939) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.553544998168945 (31.07588956522387) + | > grad_norm_1: tensor(1358.3480, device='cuda:0') (tensor(1023.6444, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.852 (0.8047759449759195) + | > loader_time: 0.0115 (0.009843720946201057) + + + --> TIME: 2024-06-25 11:36:34 -- STEP: 111/591 -- GLOBAL_STEP: 32025 + | > loss_disc: 2.6778273582458496 (2.6436846814714032) + | > loss_disc_real_0: 0.24052929878234863 (0.20017703387651362) + | > loss_disc_real_1: 0.23708800971508026 (0.22476370498403772) + | > loss_disc_real_2: 0.22142189741134644 (0.23316581265346423) + | > loss_disc_real_3: 0.233480304479599 (0.23425533521819758) + | > loss_disc_real_4: 0.22586943209171295 (0.21470410907053733) + | > loss_disc_real_5: 0.2378256767988205 (0.2378901445919329) + | > loss_0: 2.6778273582458496 (2.6436846814714032) + | > grad_norm_0: tensor(12.6806, device='cuda:0') (tensor(20.7838, device='cuda:0')) + | > loss_gen: 1.8602712154388428 (2.067391684463431) + | > loss_kl: 2.5965144634246826 (2.536145038432903) + | > loss_feat: 4.214548110961914 (4.46206659454483) + | > loss_mel: 20.54088020324707 (20.632636285042985) + | > loss_duration: 1.3160614967346191 (1.3372165501654685) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.528276443481445 (31.03545618486834) + | > grad_norm_1: tensor(501.3100, device='cuda:0') (tensor(973.1774, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.8459 (0.8109420226500915) + | > loader_time: 0.0115 (0.010135603380632833) + + + --> TIME: 2024-06-25 11:36:56 -- STEP: 136/591 -- GLOBAL_STEP: 32050 + | > loss_disc: 2.6202280521392822 (2.646079983781366) + | > loss_disc_real_0: 0.1614813506603241 (0.19961915219969611) + | > loss_disc_real_1: 0.17856809496879578 (0.22406640462577343) + | > loss_disc_real_2: 0.2233509123325348 (0.23327539992683072) + | > loss_disc_real_3: 0.25140729546546936 (0.2339952653821777) + | > loss_disc_real_4: 0.19364973902702332 (0.21670871962080984) + | > loss_disc_real_5: 0.23365283012390137 (0.23828493277816212) + | > loss_0: 2.6202280521392822 (2.646079983781366) + | > grad_norm_0: tensor(14.2643, device='cuda:0') (tensor(20.1968, device='cuda:0')) + | > loss_gen: 2.0356101989746094 (2.057219452717724) + | > loss_kl: 2.5617482662200928 (2.535952548770344) + | > loss_feat: 4.34892463684082 (4.4408667981624586) + | > loss_mel: 20.27570152282715 (20.624199278214405) + | > loss_duration: 1.3362400531768799 (1.3378344204495936) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.558225631713867 (30.99607254477108) + | > grad_norm_1: tensor(622.3932, device='cuda:0') (tensor(926.5120, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.8449 (0.8164516810108634) + | > loader_time: 0.0108 (0.010344947085661048) + + + --> TIME: 2024-06-25 11:37:17 -- STEP: 161/591 -- GLOBAL_STEP: 32075 + | > loss_disc: 2.645212173461914 (2.645237161506037) + | > loss_disc_real_0: 0.2431366890668869 (0.1992900081858132) + | > loss_disc_real_1: 0.22527186572551727 (0.22393281498680945) + | > loss_disc_real_2: 0.2277096062898636 (0.2330591922961407) + | > loss_disc_real_3: 0.23612867295742035 (0.23392653789209283) + | > loss_disc_real_4: 0.22101594507694244 (0.21804264206323565) + | > loss_disc_real_5: 0.2217819094657898 (0.2384101397687604) + | > loss_0: 2.645212173461914 (2.645237161506037) + | > grad_norm_0: tensor(18.9734, device='cuda:0') (tensor(19.6882, device='cuda:0')) + | > loss_gen: 1.9664232730865479 (2.053064706903066) + | > loss_kl: 2.608121871948242 (2.5320975291803025) + | > loss_feat: 4.364952087402344 (4.429661079963537) + | > loss_mel: 20.946033477783203 (20.597880914344564) + | > loss_duration: 1.3556894063949585 (1.3363781915688369) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.241220474243164 (30.949082439730628) + | > grad_norm_1: tensor(922.6071, device='cuda:0') (tensor(907.0167, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.8601 (0.8220584081566852) + | > loader_time: 0.0117 (0.010526674874821066) + + + --> TIME: 2024-06-25 11:37:39 -- STEP: 186/591 -- GLOBAL_STEP: 32100 + | > loss_disc: 2.627216339111328 (2.6479339317608916) + | > loss_disc_real_0: 0.17238867282867432 (0.199116572337125) + | > loss_disc_real_1: 0.2261461466550827 (0.22395257259248405) + | > loss_disc_real_2: 0.254986971616745 (0.23289202337944379) + | > loss_disc_real_3: 0.2513367533683777 (0.23388442429163123) + | > loss_disc_real_4: 0.21399974822998047 (0.21922785916956522) + | > loss_disc_real_5: 0.24977034330368042 (0.23853351896808994) + | > loss_0: 2.627216339111328 (2.6479339317608916) + | > grad_norm_0: tensor(7.4112, device='cuda:0') (tensor(19.1693, device='cuda:0')) + | > loss_gen: 2.0522353649139404 (2.047736901749847) + | > loss_kl: 2.392979621887207 (2.5286235809326185) + | > loss_feat: 4.435907363891602 (4.419694365993617) + | > loss_mel: 20.37643814086914 (20.57990880166332) + | > loss_duration: 1.3303513526916504 (1.3348677985129822) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.58791160583496 (30.910831482179702) + | > grad_norm_1: tensor(139.8009, device='cuda:0') (tensor(859.2623, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.8702 (0.8273564064374535) + | > loader_time: 0.0138 (0.01077421506245931) + + + --> TIME: 2024-06-25 11:38:02 -- STEP: 211/591 -- GLOBAL_STEP: 32125 + | > loss_disc: 2.605332374572754 (2.6464055669251234) + | > loss_disc_real_0: 0.1781935691833496 (0.1988518514904366) + | > loss_disc_real_1: 0.26551222801208496 (0.22389095153006333) + | > loss_disc_real_2: 0.2348707765340805 (0.2329722275666151) + | > loss_disc_real_3: 0.2625865042209625 (0.23381744671206903) + | > loss_disc_real_4: 0.21285495162010193 (0.2189837860128891) + | > loss_disc_real_5: 0.2506413757801056 (0.23814602090284157) + | > loss_0: 2.605332374572754 (2.6464055669251234) + | > grad_norm_0: tensor(29.7411, device='cuda:0') (tensor(19.1029, device='cuda:0')) + | > loss_gen: 2.293065071105957 (2.0474330288538996) + | > loss_kl: 2.5584280490875244 (2.5264936487821616) + | > loss_feat: 4.637945175170898 (4.4173308842555) + | > loss_mel: 20.179292678833008 (20.585217661202254) + | > loss_duration: 1.3138643503189087 (1.3341070232798142) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.982595443725586 (30.91058226671264) + | > grad_norm_1: tensor(1470.6381, device='cuda:0') (tensor(857.3004, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.8799 (0.832654976731793) + | > loader_time: 0.0124 (0.011083607425056928) + + + --> TIME: 2024-06-25 11:38:24 -- STEP: 236/591 -- GLOBAL_STEP: 32150 + | > loss_disc: 2.553889036178589 (2.6452449438935637) + | > loss_disc_real_0: 0.13555218279361725 (0.19848275014151964) + | > loss_disc_real_1: 0.2207840383052826 (0.22436734775112846) + | > loss_disc_real_2: 0.2313869446516037 (0.23338479738114243) + | > loss_disc_real_3: 0.22516542673110962 (0.23380860679988133) + | > loss_disc_real_4: 0.1504191905260086 (0.21740194258548445) + | > loss_disc_real_5: 0.20496606826782227 (0.23808788413466034) + | > loss_0: 2.553889036178589 (2.6452449438935637) + | > grad_norm_0: tensor(27.0491, device='cuda:0') (tensor(19.5588, device='cuda:0')) + | > loss_gen: 2.287717819213867 (2.056464200302705) + | > loss_kl: 2.545755624771118 (2.5265368910159105) + | > loss_feat: 4.712749004364014 (4.428298255144536) + | > loss_mel: 21.36024284362793 (20.587445937981055) + | > loss_duration: 1.3484890460968018 (1.3341477270853725) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.25495529174805 (30.9328929933451) + | > grad_norm_1: tensor(1590.9159, device='cuda:0') (tensor(883.3865, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.8917 (0.8375233937118013) + | > loader_time: 0.0131 (0.011384508367312156) + + + --> TIME: 2024-06-25 11:38:47 -- STEP: 261/591 -- GLOBAL_STEP: 32175 + | > loss_disc: 2.6575772762298584 (2.645815611799103) + | > loss_disc_real_0: 0.22737981379032135 (0.198902332463027) + | > loss_disc_real_1: 0.22198647260665894 (0.22428899546692654) + | > loss_disc_real_2: 0.2310589849948883 (0.23375560120604505) + | > loss_disc_real_3: 0.2415163367986679 (0.23402413788654794) + | > loss_disc_real_4: 0.1817764937877655 (0.21683371341091462) + | > loss_disc_real_5: 0.2408135086297989 (0.23841439278874818) + | > loss_0: 2.6575772762298584 (2.645815611799103) + | > grad_norm_0: tensor(25.6130, device='cuda:0') (tensor(19.9307, device='cuda:0')) + | > loss_gen: 1.9582167863845825 (2.058223870522217) + | > loss_kl: 2.437056303024292 (2.524157689448975) + | > loss_feat: 4.253796577453613 (4.425190604052777) + | > loss_mel: 20.0385684967041 (20.577681318553484) + | > loss_duration: 1.3634661436080933 (1.3342002643935986) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.051105499267578 (30.919453741490155) + | > grad_norm_1: tensor(1471.1309, device='cuda:0') (tensor(896.4228, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.8924 (0.8434945851906963) + | > loader_time: 0.0152 (0.011651764427565067) + + + --> TIME: 2024-06-25 11:39:10 -- STEP: 286/591 -- GLOBAL_STEP: 32200 + | > loss_disc: 2.662865161895752 (2.644740752406889) + | > loss_disc_real_0: 0.15051360428333282 (0.19859665793764006) + | > loss_disc_real_1: 0.2122126966714859 (0.22430398794529322) + | > loss_disc_real_2: 0.2279863804578781 (0.23365559220522433) + | > loss_disc_real_3: 0.19863863289356232 (0.23382748017361113) + | > loss_disc_real_4: 0.19472937285900116 (0.2160261940289211) + | > loss_disc_real_5: 0.23519115149974823 (0.23849987863869102) + | > loss_0: 2.662865161895752 (2.644740752406889) + | > grad_norm_0: tensor(13.9374, device='cuda:0') (tensor(20.0251, device='cuda:0')) + | > loss_gen: 2.1052141189575195 (2.05963514734815) + | > loss_kl: 2.486354112625122 (2.5218227534861004) + | > loss_feat: 4.473742961883545 (4.421332099220966) + | > loss_mel: 20.362823486328125 (20.555416694054248) + | > loss_duration: 1.349280834197998 (1.3349230731283874) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.777416229248047 (30.8931297422289) + | > grad_norm_1: tensor(366.0518, device='cuda:0') (tensor(903.7990, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.9136 (0.8486783862947583) + | > loader_time: 0.0138 (0.011904646466661999) + + + --> TIME: 2024-06-25 11:39:34 -- STEP: 311/591 -- GLOBAL_STEP: 32225 + | > loss_disc: 2.6468005180358887 (2.6442928413869495) + | > loss_disc_real_0: 0.1828305572271347 (0.19878806130679097) + | > loss_disc_real_1: 0.18789973855018616 (0.22443436110134676) + | > loss_disc_real_2: 0.2350705862045288 (0.23368820974489501) + | > loss_disc_real_3: 0.25492173433303833 (0.23395226023779805) + | > loss_disc_real_4: 0.23022937774658203 (0.21577765684802433) + | > loss_disc_real_5: 0.2604290544986725 (0.23850603334581738) + | > loss_0: 2.6468005180358887 (2.6442928413869495) + | > grad_norm_0: tensor(19.5029, device='cuda:0') (tensor(20.2993, device='cuda:0')) + | > loss_gen: 1.907641887664795 (2.060408457682445) + | > loss_kl: 2.4687812328338623 (2.520944524807947) + | > loss_feat: 4.289102554321289 (4.418241650345238) + | > loss_mel: 20.510889053344727 (20.536467708575394) + | > loss_duration: 1.3251357078552246 (1.3356084045489889) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.501550674438477 (30.871670719894947) + | > grad_norm_1: tensor(1267.7609, device='cuda:0') (tensor(921.0991, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.9281 (0.8550434825504706) + | > loader_time: 0.0169 (0.012163576972446258) + + + --> TIME: 2024-06-25 11:39:58 -- STEP: 336/591 -- GLOBAL_STEP: 32250 + | > loss_disc: 2.6608245372772217 (2.647760341564814) + | > loss_disc_real_0: 0.1419910192489624 (0.1989786364581614) + | > loss_disc_real_1: 0.21185661852359772 (0.22441096852223077) + | > loss_disc_real_2: 0.25382810831069946 (0.23380258138335885) + | > loss_disc_real_3: 0.2228924036026001 (0.23430163772510632) + | > loss_disc_real_4: 0.23317207396030426 (0.2164287811056489) + | > loss_disc_real_5: 0.24810287356376648 (0.23842555102670476) + | > loss_0: 2.6608245372772217 (2.647760341564814) + | > grad_norm_0: tensor(17.1099, device='cuda:0') (tensor(20.1382, device='cuda:0')) + | > loss_gen: 2.1222476959228516 (2.055375096698601) + | > loss_kl: 2.488771915435791 (2.519014108039085) + | > loss_feat: 4.192174911499023 (4.3969658081020615) + | > loss_mel: 20.670379638671875 (20.537835359573382) + | > loss_duration: 1.3397094011306763 (1.3368174930413566) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.813283920288086 (30.846007846650622) + | > grad_norm_1: tensor(53.6510, device='cuda:0') (tensor(903.6594, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.961 (0.8608023559763316) + | > loader_time: 0.0174 (0.012413970061710897) + + + --> TIME: 2024-06-25 11:40:22 -- STEP: 361/591 -- GLOBAL_STEP: 32275 + | > loss_disc: 2.6330606937408447 (2.6475647037379297) + | > loss_disc_real_0: 0.32101213932037354 (0.19952509917545855) + | > loss_disc_real_1: 0.20567479729652405 (0.2241939405325047) + | > loss_disc_real_2: 0.23595422506332397 (0.2336231182404172) + | > loss_disc_real_3: 0.24133755266666412 (0.234019159734084) + | > loss_disc_real_4: 0.20467184484004974 (0.21690425615231415) + | > loss_disc_real_5: 0.2357873171567917 (0.23838798561419808) + | > loss_0: 2.6330606937408447 (2.6475647037379297) + | > grad_norm_0: tensor(28.7910, device='cuda:0') (tensor(20.1621, device='cuda:0')) + | > loss_gen: 1.9745161533355713 (2.0529991890254764) + | > loss_kl: 2.505868911743164 (2.5184390373177146) + | > loss_feat: 4.449433326721191 (4.387165380646976) + | > loss_mel: 21.200300216674805 (20.530883292412184) + | > loss_duration: 1.4164454936981201 (1.3383768995052556) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.54656219482422 (30.827863777773533) + | > grad_norm_1: tensor(1430.6666, device='cuda:0') (tensor(898.8604, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.963 (0.8666286534549785) + | > loader_time: 0.017 (0.012642221767816512) + + + --> TIME: 2024-06-25 11:40:46 -- STEP: 386/591 -- GLOBAL_STEP: 32300 + | > loss_disc: 2.6800336837768555 (2.646893472251496) + | > loss_disc_real_0: 0.18019074201583862 (0.19945689248297507) + | > loss_disc_real_1: 0.23473207652568817 (0.22420071339514588) + | > loss_disc_real_2: 0.2359456717967987 (0.2334785817355072) + | > loss_disc_real_3: 0.2383904755115509 (0.2338561628766628) + | > loss_disc_real_4: 0.2352379858493805 (0.21632913029101228) + | > loss_disc_real_5: 0.24329505860805511 (0.23832782388814372) + | > loss_0: 2.6800336837768555 (2.646893472251496) + | > grad_norm_0: tensor(7.0068, device='cuda:0') (tensor(20.2912, device='cuda:0')) + | > loss_gen: 1.93160879611969 (2.054318253240437) + | > loss_kl: 2.3476974964141846 (2.514625580199643) + | > loss_feat: 3.8639819622039795 (4.382620271005777) + | > loss_mel: 20.075790405273438 (20.51244225279656) + | > loss_duration: 1.339118480682373 (1.3397974050724444) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.558197021484375 (30.803803765094344) + | > grad_norm_1: tensor(78.3716, device='cuda:0') (tensor(906.3205, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 1.0536 (0.8732376333345402) + | > loader_time: 0.0178 (0.012934682900423825) + + + --> TIME: 2024-06-25 11:41:12 -- STEP: 411/591 -- GLOBAL_STEP: 32325 + | > loss_disc: 2.6106059551239014 (2.6474892623233095) + | > loss_disc_real_0: 0.23701107501983643 (0.19941261673568236) + | > loss_disc_real_1: 0.22084812819957733 (0.22433302042113024) + | > loss_disc_real_2: 0.23149719834327698 (0.23352041369227017) + | > loss_disc_real_3: 0.30307644605636597 (0.23406736407691836) + | > loss_disc_real_4: 0.2153988629579544 (0.2166348296577913) + | > loss_disc_real_5: 0.23961275815963745 (0.23839914454305838) + | > loss_0: 2.6106059551239014 (2.6474892623233095) + | > grad_norm_0: tensor(21.5021, device='cuda:0') (tensor(20.2328, device='cuda:0')) + | > loss_gen: 1.8167403936386108 (2.0525644196791073) + | > loss_kl: 2.321732997894287 (2.5125318026020595) + | > loss_feat: 4.119312763214111 (4.373882302982672) + | > loss_mel: 19.765151977539062 (20.501283255806815) + | > loss_duration: 1.3783409595489502 (1.3408319100266246) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.40127944946289 (30.78109369950863) + | > grad_norm_1: tensor(1037.3640, device='cuda:0') (tensor(907.1994, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 1.0529 (0.8799617696562532) + | > loader_time: 0.0184 (0.01322102720720054) + + + --> TIME: 2024-06-25 11:41:37 -- STEP: 436/591 -- GLOBAL_STEP: 32350 + | > loss_disc: 2.6558403968811035 (2.647203820014218) + | > loss_disc_real_0: 0.21541208028793335 (0.19944839398248487) + | > loss_disc_real_1: 0.19022144377231598 (0.22407888009323987) + | > loss_disc_real_2: 0.19734960794448853 (0.23332248019789337) + | > loss_disc_real_3: 0.23395149409770966 (0.23398182811130078) + | > loss_disc_real_4: 0.22306081652641296 (0.2163347113501588) + | > loss_disc_real_5: 0.20754623413085938 (0.2384308425179862) + | > loss_0: 2.6558403968811035 (2.647203820014218) + | > grad_norm_0: tensor(17.4564, device='cuda:0') (tensor(20.4265, device='cuda:0')) + | > loss_gen: 2.1179392337799072 (2.053352841816909) + | > loss_kl: 2.3456223011016846 (2.5115327031240566) + | > loss_feat: 4.53556489944458 (4.364387170437278) + | > loss_mel: 20.590742111206055 (20.48425016490692) + | > loss_duration: 1.3695698976516724 (1.3418602711016978) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.95943832397461 (30.75538314591854) + | > grad_norm_1: tensor(1043.6047, device='cuda:0') (tensor(916.3151, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 0.9945 (0.8866285430182009) + | > loader_time: 0.0197 (0.013485806250790933) + + + --> TIME: 2024-06-25 11:42:03 -- STEP: 461/591 -- GLOBAL_STEP: 32375 + | > loss_disc: 2.6390113830566406 (2.6479262726425863) + | > loss_disc_real_0: 0.23865395784378052 (0.19943110444411286) + | > loss_disc_real_1: 0.2476043999195099 (0.22422081109241396) + | > loss_disc_real_2: 0.23364132642745972 (0.23331858560345953) + | > loss_disc_real_3: 0.22003214061260223 (0.2338962744024467) + | > loss_disc_real_4: 0.23946233093738556 (0.21695122135694012) + | > loss_disc_real_5: 0.2450997680425644 (0.23851790859357913) + | > loss_0: 2.6390113830566406 (2.6479262726425863) + | > grad_norm_0: tensor(21.3605, device='cuda:0') (tensor(20.3854, device='cuda:0')) + | > loss_gen: 2.127210855484009 (2.0500617945271786) + | > loss_kl: 2.506676197052002 (2.5099557676956623) + | > loss_feat: 4.254315376281738 (4.35068185675946) + | > loss_mel: 20.049747467041016 (20.46706206265862) + | > loss_duration: 1.3699243068695068 (1.3431021893102022) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.30787467956543 (30.72086366500358) + | > grad_norm_1: tensor(994.8178, device='cuda:0') (tensor(908.3436, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 1.0187 (0.8935236594682142) + | > loader_time: 0.0238 (0.013793824810268042) + + + --> TIME: 2024-06-25 11:42:29 -- STEP: 486/591 -- GLOBAL_STEP: 32400 + | > loss_disc: 2.625056743621826 (2.646363081755461) + | > loss_disc_real_0: 0.11054982244968414 (0.19955668386853778) + | > loss_disc_real_1: 0.1960747390985489 (0.22403676305042863) + | > loss_disc_real_2: 0.22804175317287445 (0.23310394667916828) + | > loss_disc_real_3: 0.22318106889724731 (0.23375425018646098) + | > loss_disc_real_4: 0.2049959897994995 (0.21600389833190314) + | > loss_disc_real_5: 0.2307530790567398 (0.2383987997723705) + | > loss_0: 2.625056743621826 (2.646363081755461) + | > grad_norm_0: tensor(30.9166, device='cuda:0') (tensor(20.7282, device='cuda:0')) + | > loss_gen: 2.3231897354125977 (2.0525545374846725) + | > loss_kl: 2.40278959274292 (2.506573109469788) + | > loss_feat: 4.569393634796143 (4.350855428004954) + | > loss_mel: 19.9350643157959 (20.46305173316611) + | > loss_duration: 1.3493777513504028 (1.3443500254870442) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.579816818237305 (30.717384828952113) + | > grad_norm_1: tensor(1687.5293, device='cuda:0') (tensor(917.1375, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 1.069 (0.9006651924470814) + | > loader_time: 0.0195 (0.014118653266027627) + + + --> TIME: 2024-06-25 11:42:57 -- STEP: 511/591 -- GLOBAL_STEP: 32425 + | > loss_disc: 2.604820728302002 (2.647692268375314) + | > loss_disc_real_0: 0.23168089985847473 (0.19997917025289896) + | > loss_disc_real_1: 0.17764204740524292 (0.22398193063917926) + | > loss_disc_real_2: 0.24607045948505402 (0.23312367198518577) + | > loss_disc_real_3: 0.23598775267601013 (0.23385289083954872) + | > loss_disc_real_4: 0.22938162088394165 (0.21630455075294772) + | > loss_disc_real_5: 0.2457602471113205 (0.23858399549225773) + | > loss_0: 2.604820728302002 (2.647692268375314) + | > grad_norm_0: tensor(18.2913, device='cuda:0') (tensor(20.7518, device='cuda:0')) + | > loss_gen: 2.062331199645996 (2.050465417234862) + | > loss_kl: 2.497025966644287 (2.5043857694838625) + | > loss_feat: 4.352454662322998 (4.3396784624708165) + | > loss_mel: 20.0468807220459 (20.452461772702218) + | > loss_duration: 1.3978354930877686 (1.3460990234830374) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.356529235839844 (30.693090442575354) + | > grad_norm_1: tensor(1144.5814, device='cuda:0') (tensor(914.2368, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 1.0731 (0.9098678209077121) + | > loader_time: 0.0237 (0.014436178711295823) + + + --> TIME: 2024-06-25 11:43:26 -- STEP: 536/591 -- GLOBAL_STEP: 32450 + | > loss_disc: 2.706474781036377 (2.647779474062706) + | > loss_disc_real_0: 0.2520592212677002 (0.19992367397827004) + | > loss_disc_real_1: 0.18626092374324799 (0.22396375686486264) + | > loss_disc_real_2: 0.24794386327266693 (0.23308523833306874) + | > loss_disc_real_3: 0.2767312526702881 (0.23386587441634776) + | > loss_disc_real_4: 0.23734045028686523 (0.21662942208905717) + | > loss_disc_real_5: 0.24595586955547333 (0.23851627016912644) + | > loss_0: 2.706474781036377 (2.647779474062706) + | > grad_norm_0: tensor(11.4273, device='cuda:0') (tensor(20.4981, device='cuda:0')) + | > loss_gen: 1.8017324209213257 (2.0477472105577785) + | > loss_kl: 2.3886687755584717 (2.5021010090166076) + | > loss_feat: 4.023526668548584 (4.3300116689347545) + | > loss_mel: 20.810901641845703 (20.443552643505505) + | > loss_duration: 1.4014325141906738 (1.3477024593904836) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.42626190185547 (30.67111498206409) + | > grad_norm_1: tensor(182.5643, device='cuda:0') (tensor(905.7912, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 1.1188 (0.9203059815648775) + | > loader_time: 0.0231 (0.014759876390001665) + + + --> TIME: 2024-06-25 11:43:57 -- STEP: 561/591 -- GLOBAL_STEP: 32475 + | > loss_disc: 2.589927911758423 (2.6486025727793923) + | > loss_disc_real_0: 0.13980141282081604 (0.2000260366103654) + | > loss_disc_real_1: 0.19548428058624268 (0.22401608484207838) + | > loss_disc_real_2: 0.22828294336795807 (0.23297450501544809) + | > loss_disc_real_3: 0.19937928020954132 (0.23374743581668148) + | > loss_disc_real_4: 0.19614796340465546 (0.21691416326159882) + | > loss_disc_real_5: 0.2358672320842743 (0.23855759474364194) + | > loss_0: 2.589927911758423 (2.6486025727793923) + | > grad_norm_0: tensor(25.2704, device='cuda:0') (tensor(20.4330, device='cuda:0')) + | > loss_gen: 2.071882486343384 (2.0454821656731976) + | > loss_kl: 2.377833843231201 (2.5001547557572428) + | > loss_feat: 4.1552510261535645 (4.317050948287504) + | > loss_mel: 20.23080825805664 (20.42803555928875) + | > loss_duration: 1.350559949874878 (1.3493967241144436) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.18633460998535 (30.640120133996646) + | > grad_norm_1: tensor(1167.6747, device='cuda:0') (tensor(900.7482, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 1.1853 (0.9322092443863976) + | > loader_time: 0.0253 (0.015142569992419017) + + + --> TIME: 2024-06-25 11:44:29 -- STEP: 586/591 -- GLOBAL_STEP: 32500 + | > loss_disc: 2.7190184593200684 (2.6499055579253845) + | > loss_disc_real_0: 0.35005462169647217 (0.20022338712683313) + | > loss_disc_real_1: 0.24815019965171814 (0.2239720538105981) + | > loss_disc_real_2: 0.21429511904716492 (0.2329115114928106) + | > loss_disc_real_3: 0.22214512526988983 (0.23381611847226524) + | > loss_disc_real_4: 0.2607097327709198 (0.2175093862790703) + | > loss_disc_real_5: 0.24757929146289825 (0.23853314128853761) + | > loss_0: 2.7190184593200684 (2.6499055579253845) + | > grad_norm_0: tensor(38.1984, device='cuda:0') (tensor(20.3216, device='cuda:0')) + | > loss_gen: 1.7861732244491577 (2.0426345502150327) + | > loss_kl: 2.3996388912200928 (2.497759190028846) + | > loss_feat: 3.964473247528076 (4.303224437472767) + | > loss_mel: 20.030794143676758 (20.41153524353235) + | > loss_duration: 1.382595419883728 (1.3510509668356732) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.56367301940918 (30.60620437465837) + | > grad_norm_1: tensor(788.8058, device='cuda:0') (tensor(892.0866, device='cuda:0')) + | > current_lr_0: 0.00019865446220135974 + | > current_lr_1: 0.00019865446220135974 + | > step_time: 1.2668 (0.9456247424916603) + | > loader_time: 0.0221 (0.015540155941308966) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036308156683089884 (+3.893324669371244e-05) + | > avg_loss_disc: 2.619255806537384 (-0.08363142926642375) + | > avg_loss_disc_real_0: 0.30300283939280415 (+0.008141910459132873) + | > avg_loss_disc_real_1: 0.23715642379953505 (+0.01599885333091655) + | > avg_loss_disc_real_2: 0.2058600716768427 (-0.027584693216263) + | > avg_loss_disc_real_3: 0.22415869762288762 (-0.008422990428640503) + | > avg_loss_disc_real_4: 0.2328105499135687 (-0.05584806235546769) + | > avg_loss_disc_real_5: 0.24522868337783407 (-0.0012510162084660714) + | > avg_loss_0: 2.619255806537384 (-0.08363142926642375) + | > avg_loss_gen: 2.173362047114271 (+0.03749862883953359) + | > avg_loss_kl: 2.59719562023244 (+0.06552116414333797) + | > avg_loss_feat: 4.252786585625181 (+0.22334680151432096) + | > avg_loss_mel: 20.490263837449096 (+0.1907976028767031) + | > avg_loss_duration: 1.366103080993003 (-0.005025577037892415) + | > avg_loss_1: 30.87971123228682 (+0.5121384478629913) + + + > EPOCH: 56/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 11:44:43)  + + --> TIME: 2024-06-25 11:45:00 -- STEP: 20/591 -- GLOBAL_STEP: 32525 + | > loss_disc: 2.53554368019104 (2.606320536136627) + | > loss_disc_real_0: 0.19089215993881226 (0.19780659973621367) + | > loss_disc_real_1: 0.2415464222431183 (0.22245949655771255) + | > loss_disc_real_2: 0.24985089898109436 (0.2301186926662922) + | > loss_disc_real_3: 0.2318439781665802 (0.23654889240860938) + | > loss_disc_real_4: 0.2502949833869934 (0.20826752930879594) + | > loss_disc_real_5: 0.2956632077693939 (0.2405618965625763) + | > loss_0: 2.53554368019104 (2.606320536136627) + | > grad_norm_0: tensor(19.4348, device='cuda:0') (tensor(23.0605, device='cuda:0')) + | > loss_gen: 2.2066097259521484 (2.1545780479907988) + | > loss_kl: 2.605971574783325 (2.598280119895935) + | > loss_feat: 5.123850345611572 (4.772073698043823) + | > loss_mel: 20.591167449951172 (20.595776748657226) + | > loss_duration: 1.3043328523635864 (1.3151184141635894) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.831932067871094 (31.435826969146728) + | > grad_norm_1: tensor(1436.9927, device='cuda:0') (tensor(1113.9036, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.7738 (0.7758675575256347) + | > loader_time: 0.0088 (0.00896984338760376) + + + --> TIME: 2024-06-25 11:45:20 -- STEP: 45/591 -- GLOBAL_STEP: 32550 + | > loss_disc: 2.4534876346588135 (2.6215725209977894) + | > loss_disc_real_0: 0.1647930145263672 (0.1994834777381685) + | > loss_disc_real_1: 0.1738923043012619 (0.22366966572072772) + | > loss_disc_real_2: 0.21445998549461365 (0.23182847566074796) + | > loss_disc_real_3: 0.20678415894508362 (0.23499904407395256) + | > loss_disc_real_4: 0.19707903265953064 (0.205263951420784) + | > loss_disc_real_5: 0.22535616159439087 (0.23976736300521426) + | > loss_0: 2.4534876346588135 (2.6215725209977894) + | > grad_norm_0: tensor(28.9019, device='cuda:0') (tensor(21.6064, device='cuda:0')) + | > loss_gen: 2.0677173137664795 (2.1341636631223895) + | > loss_kl: 2.4676525592803955 (2.5633582433064777) + | > loss_feat: 4.7618184089660645 (4.673720328013102) + | > loss_mel: 20.842538833618164 (20.589580790201822) + | > loss_duration: 1.3431718349456787 (1.322275691562229) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.482898712158203 (31.28309877183702) + | > grad_norm_1: tensor(1418.4268, device='cuda:0') (tensor(1135.6526, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.8108 (0.7881716145409479) + | > loader_time: 0.0096 (0.009344371159871419) + + + --> TIME: 2024-06-25 11:45:41 -- STEP: 70/591 -- GLOBAL_STEP: 32575 + | > loss_disc: 2.5591347217559814 (2.621210452488491) + | > loss_disc_real_0: 0.13200698792934418 (0.19946294384343283) + | > loss_disc_real_1: 0.25721505284309387 (0.2242094459278243) + | > loss_disc_real_2: 0.2032369077205658 (0.23159088662692479) + | > loss_disc_real_3: 0.23999997973442078 (0.23375295741217478) + | > loss_disc_real_4: 0.2204495519399643 (0.2049090095928737) + | > loss_disc_real_5: 0.22977474331855774 (0.2387140018599374) + | > loss_0: 2.5591347217559814 (2.621210452488491) + | > grad_norm_0: tensor(29.4744, device='cuda:0') (tensor(23.7626, device='cuda:0')) + | > loss_gen: 2.280221462249756 (2.140250343935831) + | > loss_kl: 2.5939199924468994 (2.5487551280430387) + | > loss_feat: 4.819027423858643 (4.6331987312861855) + | > loss_mel: 20.570589065551758 (20.52144208635603) + | > loss_duration: 1.3334094285964966 (1.328400751522609) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.59716796875 (31.172047206333705) + | > grad_norm_1: tensor(1599.6960, device='cuda:0') (tensor(1151.8632, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.814 (0.7965568678719657) + | > loader_time: 0.01 (0.009710489000592912) + + + --> TIME: 2024-06-25 11:46:02 -- STEP: 95/591 -- GLOBAL_STEP: 32600 + | > loss_disc: 2.6887643337249756 (2.625991545225446) + | > loss_disc_real_0: 0.1668570339679718 (0.19939377402004438) + | > loss_disc_real_1: 0.21141092479228973 (0.22361073901778775) + | > loss_disc_real_2: 0.23304934799671173 (0.23233963549137115) + | > loss_disc_real_3: 0.2225559800863266 (0.2334107091552333) + | > loss_disc_real_4: 0.2041766494512558 (0.20657893450636614) + | > loss_disc_real_5: 0.20896658301353455 (0.23854928644079912) + | > loss_0: 2.6887643337249756 (2.625991545225446) + | > grad_norm_0: tensor(8.9223, device='cuda:0') (tensor(23.1786, device='cuda:0')) + | > loss_gen: 1.7926232814788818 (2.121275975829678) + | > loss_kl: 2.505035638809204 (2.54555898967542) + | > loss_feat: 4.172940731048584 (4.591540738155969) + | > loss_mel: 21.150527954101562 (20.5885691994115) + | > loss_duration: 1.3666191101074219 (1.335369472754629) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.987747192382812 (31.182314380846524) + | > grad_norm_1: tensor(114.4051, device='cuda:0') (tensor(1091.7344, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.8309 (0.8053570195248253) + | > loader_time: 0.0124 (0.010157361783479384) + + + --> TIME: 2024-06-25 11:46:23 -- STEP: 120/591 -- GLOBAL_STEP: 32625 + | > loss_disc: 2.6875391006469727 (2.6311977366606403) + | > loss_disc_real_0: 0.18975825607776642 (0.1989543491353591) + | > loss_disc_real_1: 0.21939653158187866 (0.22353960536420345) + | > loss_disc_real_2: 0.24417993426322937 (0.2322868221749862) + | > loss_disc_real_3: 0.2921791076660156 (0.23347261312107245) + | > loss_disc_real_4: 0.2210817039012909 (0.21150734064479668) + | > loss_disc_real_5: 0.22672750055789948 (0.2376317262649536) + | > loss_0: 2.6875391006469727 (2.6311977366606403) + | > grad_norm_0: tensor(9.5574, device='cuda:0') (tensor(20.9842, device='cuda:0')) + | > loss_gen: 1.8092542886734009 (2.096562076608341) + | > loss_kl: 2.527099132537842 (2.5366149266560853) + | > loss_feat: 4.402528762817383 (4.537024786074958) + | > loss_mel: 21.554040908813477 (20.598859167099008) + | > loss_duration: 1.3664212226867676 (1.3363258798917135) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.659343719482422 (31.105386813481648) + | > grad_norm_1: tensor(263.9612, device='cuda:0') (tensor(953.7354, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.8714 (0.8116072634855906) + | > loader_time: 0.0123 (0.010425148407618197) + + + --> TIME: 2024-06-25 11:46:45 -- STEP: 145/591 -- GLOBAL_STEP: 32650 + | > loss_disc: 2.6784229278564453 (2.6371008313935387) + | > loss_disc_real_0: 0.2581745386123657 (0.19999360699078128) + | > loss_disc_real_1: 0.2422112226486206 (0.22321144013569272) + | > loss_disc_real_2: 0.23529164493083954 (0.23261923779701366) + | > loss_disc_real_3: 0.2602277100086212 (0.23283477980515052) + | > loss_disc_real_4: 0.2309693545103073 (0.21479579713837854) + | > loss_disc_real_5: 0.21741239726543427 (0.23778828988815176) + | > loss_0: 2.6784229278564453 (2.6371008313935387) + | > grad_norm_0: tensor(17.4072, device='cuda:0') (tensor(20.0676, device='cuda:0')) + | > loss_gen: 1.88692045211792 (2.0815616903633916) + | > loss_kl: 2.516976833343506 (2.531748985422068) + | > loss_feat: 4.156307697296143 (4.499054594697625) + | > loss_mel: 20.553287506103516 (20.603785481946232) + | > loss_duration: 1.3623933792114258 (1.336886080380144) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.475887298583984 (31.053036867339035) + | > grad_norm_1: tensor(420.8320, device='cuda:0') (tensor(877.5630, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.8518 (0.8171899779089568) + | > loader_time: 0.0135 (0.010655016734682275) + + + --> TIME: 2024-06-25 11:47:07 -- STEP: 170/591 -- GLOBAL_STEP: 32675 + | > loss_disc: 2.7209606170654297 (2.643034639078028) + | > loss_disc_real_0: 0.15901336073875427 (0.19988183221396275) + | > loss_disc_real_1: 0.24130479991436005 (0.22321395383161657) + | > loss_disc_real_2: 0.23687544465065002 (0.2323210150879972) + | > loss_disc_real_3: 0.2682132422924042 (0.2327812993351151) + | > loss_disc_real_4: 0.21504005789756775 (0.21726449394927305) + | > loss_disc_real_5: 0.2320508509874344 (0.23753574876224293) + | > loss_0: 2.7209606170654297 (2.643034639078028) + | > grad_norm_0: tensor(7.4177, device='cuda:0') (tensor(18.8353, device='cuda:0')) + | > loss_gen: 1.9842302799224854 (2.065213355597329) + | > loss_kl: 2.515767812728882 (2.5277277722078204) + | > loss_feat: 4.424973487854004 (4.463040333635666) + | > loss_mel: 20.700355529785156 (20.57407888524673) + | > loss_duration: 1.3079551458358765 (1.3349450805607963) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.93328285217285 (30.965005493164064) + | > grad_norm_1: tensor(112.1124, device='cuda:0') (tensor(802.0221, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.8761 (0.8233053726308488) + | > loader_time: 0.0121 (0.010880262711468854) + + + --> TIME: 2024-06-25 11:47:29 -- STEP: 195/591 -- GLOBAL_STEP: 32700 + | > loss_disc: 2.5636584758758545 (2.64970103655106) + | > loss_disc_real_0: 0.17085391283035278 (0.200213910639286) + | > loss_disc_real_1: 0.20160381495952606 (0.2236864102956576) + | > loss_disc_real_2: 0.21618297696113586 (0.233009177675614) + | > loss_disc_real_3: 0.22185057401657104 (0.23312332782989892) + | > loss_disc_real_4: 0.2177029252052307 (0.2192454396150051) + | > loss_disc_real_5: 0.25993889570236206 (0.2382315903902054) + | > loss_0: 2.5636584758758545 (2.64970103655106) + | > grad_norm_0: tensor(18.5636, device='cuda:0') (tensor(18.3062, device='cuda:0')) + | > loss_gen: 2.0472264289855957 (2.054006966566428) + | > loss_kl: 2.57450532913208 (2.5221844929915216) + | > loss_feat: 4.459064960479736 (4.432929837398037) + | > loss_mel: 20.166542053222656 (20.557295853052395) + | > loss_duration: 1.3187532424926758 (1.3340763526085089) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.566089630126953 (30.90049358270107) + | > grad_norm_1: tensor(1004.4950, device='cuda:0') (tensor(765.9269, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.8548 (0.828692063307151) + | > loader_time: 0.0126 (0.011086913866874491) + + + --> TIME: 2024-06-25 11:47:51 -- STEP: 220/591 -- GLOBAL_STEP: 32725 + | > loss_disc: 2.6243577003479004 (2.6461416190320795) + | > loss_disc_real_0: 0.14184749126434326 (0.1998239719393578) + | > loss_disc_real_1: 0.2458135336637497 (0.22423334995454008) + | > loss_disc_real_2: 0.21103444695472717 (0.23258928514339708) + | > loss_disc_real_3: 0.23419071733951569 (0.23316077291965484) + | > loss_disc_real_4: 0.2799580991268158 (0.2161320921033621) + | > loss_disc_real_5: 0.27596190571784973 (0.2380846487527544) + | > loss_0: 2.6243577003479004 (2.6461416190320795) + | > grad_norm_0: tensor(21.6956, device='cuda:0') (tensor(19.0241, device='cuda:0')) + | > loss_gen: 2.38923978805542 (2.069639898972078) + | > loss_kl: 2.4986660480499268 (2.515020267529922) + | > loss_feat: 4.6243743896484375 (4.460262897881593) + | > loss_mel: 21.162324905395508 (20.568049040707685) + | > loss_duration: 1.3189269304275513 (1.3333172917366023) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.993534088134766 (30.94628948731856) + | > grad_norm_1: tensor(999.6183, device='cuda:0') (tensor(780.9329, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.8904 (0.8339041428132492) + | > loader_time: 0.0144 (0.011392969434911547) + + + --> TIME: 2024-06-25 11:48:14 -- STEP: 245/591 -- GLOBAL_STEP: 32750 + | > loss_disc: 2.6721858978271484 (2.6495772147665217) + | > loss_disc_real_0: 0.21556374430656433 (0.20011898434283779) + | > loss_disc_real_1: 0.1999705582857132 (0.22444642210493282) + | > loss_disc_real_2: 0.22250613570213318 (0.23260195620205937) + | > loss_disc_real_3: 0.2545100450515747 (0.23332994215342465) + | > loss_disc_real_4: 0.23805415630340576 (0.21878889078388408) + | > loss_disc_real_5: 0.24996936321258545 (0.23789749072522534) + | > loss_0: 2.6721858978271484 (2.6495772147665217) + | > grad_norm_0: tensor(9.6548, device='cuda:0') (tensor(18.5573, device='cuda:0')) + | > loss_gen: 1.9278405904769897 (2.0621524304759755) + | > loss_kl: 2.559544086456299 (2.5131670951843272) + | > loss_feat: 4.117522716522217 (4.443233993102092) + | > loss_mel: 20.0633544921875 (20.574216648023963) + | > loss_duration: 1.3590493202209473 (1.3335856486339954) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.027311325073242 (30.926355867969747) + | > grad_norm_1: tensor(301.0988, device='cuda:0') (tensor(755.7049, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.9031 (0.8396172114780972) + | > loader_time: 0.015 (0.011713053255665051) + + + --> TIME: 2024-06-25 11:48:37 -- STEP: 270/591 -- GLOBAL_STEP: 32775 + | > loss_disc: 2.59242844581604 (2.6488460196389094) + | > loss_disc_real_0: 0.18810619413852692 (0.20016625244860292) + | > loss_disc_real_1: 0.21234282851219177 (0.2241257828142908) + | > loss_disc_real_2: 0.23536522686481476 (0.23241878471992633) + | > loss_disc_real_3: 0.20025889575481415 (0.23287023935053083) + | > loss_disc_real_4: 0.19456081092357635 (0.21920186383304774) + | > loss_disc_real_5: 0.20360763370990753 (0.2379174233034805) + | > loss_0: 2.59242844581604 (2.6488460196389094) + | > grad_norm_0: tensor(20.6740, device='cuda:0') (tensor(18.3380, device='cuda:0')) + | > loss_gen: 1.9382328987121582 (2.0569728776260665) + | > loss_kl: 2.4105889797210693 (2.5115405232818047) + | > loss_feat: 4.666679859161377 (4.432910611894395) + | > loss_mel: 20.66594696044922 (20.554500904789684) + | > loss_duration: 1.3558993339538574 (1.3336132265903329) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.0373477935791 (30.889538213941787) + | > grad_norm_1: tensor(1261.8239, device='cuda:0') (tensor(749.4190, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.877 (0.8457978054329203) + | > loader_time: 0.0141 (0.011981374246102785) + + + --> TIME: 2024-06-25 11:49:00 -- STEP: 295/591 -- GLOBAL_STEP: 32800 + | > loss_disc: 2.613755702972412 (2.6480437028206) + | > loss_disc_real_0: 0.281535267829895 (0.20021813989190732) + | > loss_disc_real_1: 0.21055760979652405 (0.22406417631496817) + | > loss_disc_real_2: 0.22818252444267273 (0.23264967662803196) + | > loss_disc_real_3: 0.22672149538993835 (0.2332020758572272) + | > loss_disc_real_4: 0.2501756250858307 (0.21801119166410576) + | > loss_disc_real_5: 0.2571021318435669 (0.23804475079148502) + | > loss_0: 2.613755702972412 (2.6480437028206) + | > grad_norm_0: tensor(27.3528, device='cuda:0') (tensor(18.6094, device='cuda:0')) + | > loss_gen: 2.1281394958496094 (2.0610512450589984) + | > loss_kl: 2.581775188446045 (2.5089838634103043) + | > loss_feat: 4.55158805847168 (4.433940396066439) + | > loss_mel: 21.16010093688965 (20.550989319106286) + | > loss_duration: 1.3326776027679443 (1.334338047140736) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.754281997680664 (30.889302929377152) + | > grad_norm_1: tensor(1282.5178, device='cuda:0') (tensor(770.6639, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.9101 (0.8512429762694799) + | > loader_time: 0.0135 (0.012213711819406278) + + + --> TIME: 2024-06-25 11:49:24 -- STEP: 320/591 -- GLOBAL_STEP: 32825 + | > loss_disc: 2.6306092739105225 (2.647441305965185) + | > loss_disc_real_0: 0.14578954875469208 (0.20005273956339806) + | > loss_disc_real_1: 0.24404792487621307 (0.22389499377459288) + | > loss_disc_real_2: 0.2311202436685562 (0.2328109207563102) + | > loss_disc_real_3: 0.24556231498718262 (0.23329813019372528) + | > loss_disc_real_4: 0.21565338969230652 (0.21796214750502257) + | > loss_disc_real_5: 0.2549396753311157 (0.23817057493142785) + | > loss_0: 2.6306092739105225 (2.647441305965185) + | > grad_norm_0: tensor(26.0707, device='cuda:0') (tensor(18.8644, device='cuda:0')) + | > loss_gen: 1.9709581136703491 (2.060080309212208) + | > loss_kl: 2.4082086086273193 (2.5092427827417856) + | > loss_feat: 4.12959098815918 (4.42905031144619) + | > loss_mel: 20.51997947692871 (20.54123758673669) + | > loss_duration: 1.3175939321517944 (1.3352203726768495) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.346332550048828 (30.87483141422272) + | > grad_norm_1: tensor(1168.4447, device='cuda:0') (tensor(783.4286, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.9058 (0.8560129746794705) + | > loader_time: 0.0171 (0.012490077316761017) + + + --> TIME: 2024-06-25 11:49:47 -- STEP: 345/591 -- GLOBAL_STEP: 32850 + | > loss_disc: 2.6272783279418945 (2.644933410312819) + | > loss_disc_real_0: 0.19778668880462646 (0.1996400858828987) + | > loss_disc_real_1: 0.26314011216163635 (0.22406156685041345) + | > loss_disc_real_2: 0.23349276185035706 (0.23273236198701722) + | > loss_disc_real_3: 0.24223193526268005 (0.23324674782545674) + | > loss_disc_real_4: 0.2631501853466034 (0.21668162931134735) + | > loss_disc_real_5: 0.25824013352394104 (0.23793667265470478) + | > loss_0: 2.6272783279418945 (2.644933410312819) + | > grad_norm_0: tensor(30.0413, device='cuda:0') (tensor(19.2477, device='cuda:0')) + | > loss_gen: 2.1142985820770264 (2.0638042163157815) + | > loss_kl: 2.5220754146575928 (2.507980952746628) + | > loss_feat: 4.3548078536987305 (4.43373674447986) + | > loss_mel: 19.497426986694336 (20.530635142671894) + | > loss_duration: 1.386089563369751 (1.3367090684780172) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.874696731567383 (30.87286617997764) + | > grad_norm_1: tensor(1259.8348, device='cuda:0') (tensor(805.6726, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.9446 (0.8612463225489081) + | > loader_time: 0.0172 (0.01278963227202927) + + + --> TIME: 2024-06-25 11:50:11 -- STEP: 370/591 -- GLOBAL_STEP: 32875 + | > loss_disc: 2.5653045177459717 (2.6443111580771377) + | > loss_disc_real_0: 0.21166551113128662 (0.199265281312369) + | > loss_disc_real_1: 0.17816589772701263 (0.2239106968850703) + | > loss_disc_real_2: 0.20768588781356812 (0.23253005420839465) + | > loss_disc_real_3: 0.21523670852184296 (0.23305308069731742) + | > loss_disc_real_4: 0.20905376970767975 (0.217025419082996) + | > loss_disc_real_5: 0.23996502161026 (0.2379657417938516) + | > loss_0: 2.5653045177459717 (2.6443111580771377) + | > grad_norm_0: tensor(21.8825, device='cuda:0') (tensor(19.0690, device='cuda:0')) + | > loss_gen: 2.012465715408325 (2.05960239075326) + | > loss_kl: 2.39793062210083 (2.505160664867712) + | > loss_feat: 4.561691761016846 (4.415101139609881) + | > loss_mel: 20.258651733398438 (20.521740212311613) + | > loss_duration: 1.3439632654190063 (1.3382844341767803) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.574703216552734 (30.83988890776763) + | > grad_norm_1: tensor(898.3228, device='cuda:0') (tensor(802.2829, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.9393 (0.8665877786842555) + | > loader_time: 0.0172 (0.013082212370795173) + + + --> TIME: 2024-06-25 11:50:36 -- STEP: 395/591 -- GLOBAL_STEP: 32900 + | > loss_disc: 2.718273401260376 (2.6454697270936602) + | > loss_disc_real_0: 0.1919139325618744 (0.19933785562273823) + | > loss_disc_real_1: 0.19884677231311798 (0.22381714465497415) + | > loss_disc_real_2: 0.2544609010219574 (0.23280519545078276) + | > loss_disc_real_3: 0.25429442524909973 (0.2331742563956901) + | > loss_disc_real_4: 0.24396105110645294 (0.21726874824188933) + | > loss_disc_real_5: 0.27880969643592834 (0.23823294979107532) + | > loss_0: 2.718273401260376 (2.6454697270936602) + | > grad_norm_0: tensor(16.5536, device='cuda:0') (tensor(19.2140, device='cuda:0')) + | > loss_gen: 1.9160704612731934 (2.058039237879504) + | > loss_kl: 2.4746968746185303 (2.50359795908385) + | > loss_feat: 3.8365392684936523 (4.404142689403102) + | > loss_mel: 20.470947265625 (20.514570583874672) + | > loss_duration: 1.3627135753631592 (1.3392786880082728) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.06096839904785 (30.819629224946226) + | > grad_norm_1: tensor(858.4593, device='cuda:0') (tensor(809.9294, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.9547 (0.8725162976904763) + | > loader_time: 0.0186 (0.013387755502628374) + + + --> TIME: 2024-06-25 11:51:01 -- STEP: 420/591 -- GLOBAL_STEP: 32925 + | > loss_disc: 2.596957206726074 (2.64456296477999) + | > loss_disc_real_0: 0.1723776012659073 (0.19902439862489701) + | > loss_disc_real_1: 0.22348539531230927 (0.22387118807860784) + | > loss_disc_real_2: 0.23802682757377625 (0.23268034777470997) + | > loss_disc_real_3: 0.2428368180990219 (0.23327280325548994) + | > loss_disc_real_4: 0.2522871494293213 (0.2169296129651013) + | > loss_disc_real_5: 0.24855159223079681 (0.23826229841936203) + | > loss_0: 2.596957206726074 (2.64456296477999) + | > grad_norm_0: tensor(38.0401, device='cuda:0') (tensor(19.4791, device='cuda:0')) + | > loss_gen: 2.1901309490203857 (2.0594715731484596) + | > loss_kl: 2.4259886741638184 (2.502423357963564) + | > loss_feat: 4.521710395812988 (4.401358664035801) + | > loss_mel: 20.025354385375977 (20.51015343893143) + | > loss_duration: 1.3599426746368408 (1.340200228918167) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.52312660217285 (30.8136073248727) + | > grad_norm_1: tensor(1558.9509, device='cuda:0') (tensor(826.8704, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.9754 (0.8786301896685649) + | > loader_time: 0.0185 (0.013657002789633615) + + + --> TIME: 2024-06-25 11:51:27 -- STEP: 445/591 -- GLOBAL_STEP: 32950 + | > loss_disc: 2.699455976486206 (2.6443705387329786) + | > loss_disc_real_0: 0.22821804881095886 (0.19914424298854355) + | > loss_disc_real_1: 0.21749472618103027 (0.22395551047298345) + | > loss_disc_real_2: 0.21973110735416412 (0.23247465986884042) + | > loss_disc_real_3: 0.2190551459789276 (0.233257707417681) + | > loss_disc_real_4: 0.20255112648010254 (0.21679486761267266) + | > loss_disc_real_5: 0.24939332902431488 (0.2382959484384301) + | > loss_0: 2.699455976486206 (2.6443705387329786) + | > grad_norm_0: tensor(7.8573, device='cuda:0') (tensor(19.5709, device='cuda:0')) + | > loss_gen: 1.8670544624328613 (2.058462747563143) + | > loss_kl: 2.3958349227905273 (2.4991624312454417) + | > loss_feat: 4.163245677947998 (4.396506312188142) + | > loss_mel: 20.386869430541992 (20.50032299770399) + | > loss_duration: 1.3807373046875 (1.3412075476700005) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.193740844726562 (30.795662076285716) + | > grad_norm_1: tensor(59.1988, device='cuda:0') (tensor(832.5560, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 1.0187 (0.885781471916799) + | > loader_time: 0.0203 (0.013952127199494437) + + + --> TIME: 2024-06-25 11:51:53 -- STEP: 470/591 -- GLOBAL_STEP: 32975 + | > loss_disc: 2.6208410263061523 (2.644560055022544) + | > loss_disc_real_0: 0.17991401255130768 (0.1994531585181013) + | > loss_disc_real_1: 0.22324365377426147 (0.2237370011654306) + | > loss_disc_real_2: 0.2485233098268509 (0.2324781910853183) + | > loss_disc_real_3: 0.22286967933177948 (0.23329079522097368) + | > loss_disc_real_4: 0.18713241815567017 (0.2165034734980857) + | > loss_disc_real_5: 0.24519748985767365 (0.23825923372456367) + | > loss_0: 2.6208410263061523 (2.644560055022544) + | > grad_norm_0: tensor(18.1724, device='cuda:0') (tensor(19.9124, device='cuda:0')) + | > loss_gen: 1.9310414791107178 (2.05796964016367) + | > loss_kl: 2.48012113571167 (2.4975132110271065) + | > loss_feat: 4.290399551391602 (4.388293886184697) + | > loss_mel: 20.08803367614746 (20.489738671323096) + | > loss_duration: 1.395904779434204 (1.342427898214219) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.185501098632812 (30.775943354342846) + | > grad_norm_1: tensor(116.5774, device='cuda:0') (tensor(842.2874, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 0.994 (0.8930060569276204) + | > loader_time: 0.0177 (0.014248806364992832) + + + --> TIME: 2024-06-25 11:52:20 -- STEP: 495/591 -- GLOBAL_STEP: 33000 + | > loss_disc: 2.6718735694885254 (2.6435980700483226) + | > loss_disc_real_0: 0.2516208291053772 (0.19929199414421814) + | > loss_disc_real_1: 0.2359144687652588 (0.22360617220401763) + | > loss_disc_real_2: 0.2496488392353058 (0.23238387357706974) + | > loss_disc_real_3: 0.21883651614189148 (0.23325922281453107) + | > loss_disc_real_4: 0.21729953587055206 (0.21671561537065892) + | > loss_disc_real_5: 0.2424737513065338 (0.23810530202557342) + | > loss_0: 2.6718735694885254 (2.6435980700483226) + | > grad_norm_0: tensor(24.7372, device='cuda:0') (tensor(19.9007, device='cuda:0')) + | > loss_gen: 1.9223854541778564 (2.0571185981384477) + | > loss_kl: 2.5264437198638916 (2.4952290674652735) + | > loss_feat: 3.9404983520507812 (4.382307422040694) + | > loss_mel: 20.142921447753906 (20.48189175923666) + | > loss_duration: 1.3642737865447998 (1.3436611298358807) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.896522521972656 (30.760208041258533) + | > grad_norm_1: tensor(709.8832, device='cuda:0') (tensor(844.6827, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 1.115 (0.9009255510388001) + | > loader_time: 0.0185 (0.014539046721024946) + + + --> TIME: 2024-06-25 11:52:48 -- STEP: 520/591 -- GLOBAL_STEP: 33025 + | > loss_disc: 2.697032928466797 (2.644062794630344) + | > loss_disc_real_0: 0.2602202892303467 (0.1994352198564089) + | > loss_disc_real_1: 0.22216998040676117 (0.22378254300699785) + | > loss_disc_real_2: 0.2139122635126114 (0.23236648102219287) + | > loss_disc_real_3: 0.19465278089046478 (0.23324700126854275) + | > loss_disc_real_4: 0.1552223563194275 (0.21657538335197246) + | > loss_disc_real_5: 0.21376459300518036 (0.23812946998156034) + | > loss_0: 2.697032928466797 (2.644062794630344) + | > grad_norm_0: tensor(22.7936, device='cuda:0') (tensor(20.1263, device='cuda:0')) + | > loss_gen: 1.9780516624450684 (2.0575145654953486) + | > loss_kl: 2.463296890258789 (2.49403023169591) + | > loss_feat: 4.217895030975342 (4.3768362879753155) + | > loss_mel: 20.670639038085938 (20.476625141730683) + | > loss_duration: 1.3763911724090576 (1.3453836415822693) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.706274032592773 (30.750389928084154) + | > grad_norm_1: tensor(1190.7166, device='cuda:0') (tensor(848.8706, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 1.1554 (0.9109943119379192) + | > loader_time: 0.0203 (0.014842794950191791) + + + --> TIME: 2024-06-25 11:53:17 -- STEP: 545/591 -- GLOBAL_STEP: 33050 + | > loss_disc: 2.5903210639953613 (2.643885849593976) + | > loss_disc_real_0: 0.1482180505990982 (0.19924442092759892) + | > loss_disc_real_1: 0.21952331066131592 (0.22366380948539177) + | > loss_disc_real_2: 0.20698219537734985 (0.23235900585257677) + | > loss_disc_real_3: 0.22616977989673615 (0.23330168015913136) + | > loss_disc_real_4: 0.22018396854400635 (0.21660061761600158) + | > loss_disc_real_5: 0.21957065165042877 (0.23813604012541814) + | > loss_0: 2.5903210639953613 (2.643885849593976) + | > grad_norm_0: tensor(15.0369, device='cuda:0') (tensor(20.1681, device='cuda:0')) + | > loss_gen: 2.0281176567077637 (2.056338627185301) + | > loss_kl: 2.4375131130218506 (2.492008745560953) + | > loss_feat: 4.165498733520508 (4.366006810949488) + | > loss_mel: 20.300704956054688 (20.459906835293566) + | > loss_duration: 1.3553805351257324 (1.3469788177297755) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.287216186523438 (30.721239884402774) + | > grad_norm_1: tensor(793.4300, device='cuda:0') (tensor(847.7660, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 1.1891 (0.9216892601153176) + | > loader_time: 0.0241 (0.015169329599502984) + + + --> TIME: 2024-06-25 11:53:48 -- STEP: 570/591 -- GLOBAL_STEP: 33075 + | > loss_disc: 2.628281831741333 (2.6451788596939636) + | > loss_disc_real_0: 0.22634068131446838 (0.19931530932846822) + | > loss_disc_real_1: 0.20774486660957336 (0.22364015035461962) + | > loss_disc_real_2: 0.20275771617889404 (0.2323413203682816) + | > loss_disc_real_3: 0.22727812826633453 (0.2332597518699211) + | > loss_disc_real_4: 0.2095792442560196 (0.21727641524191482) + | > loss_disc_real_5: 0.21727387607097626 (0.23814100897625873) + | > loss_0: 2.628281831741333 (2.6451788596939636) + | > grad_norm_0: tensor(13.2591, device='cuda:0') (tensor(19.9608, device='cuda:0')) + | > loss_gen: 2.050133228302002 (2.05234303411685) + | > loss_kl: 2.4840753078460693 (2.4902303683130382) + | > loss_feat: 4.452577590942383 (4.351203942717171) + | > loss_mel: 20.812604904174805 (20.44060219881828) + | > loss_duration: 1.3775817155838013 (1.3486555938135119) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.176973342895508 (30.68303518797222) + | > grad_norm_1: tensor(589.9496, device='cuda:0') (tensor(832.9968, device='cuda:0')) + | > current_lr_0: 0.00019862963039358455 + | > current_lr_1: 0.00019862963039358455 + | > step_time: 1.2243 (0.9335004179101245) + | > loader_time: 0.0246 (0.01552942301097669) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036078463209436294 (-2.2969347365358987e-05) + | > avg_loss_disc: 2.7140709187122103 (+0.09481511217482641) + | > avg_loss_disc_real_0: 0.11357208499883084 (-0.1894307543939733) + | > avg_loss_disc_real_1: 0.20078699766321384 (-0.036369426136321215) + | > avg_loss_disc_real_2: 0.24706865688587756 (+0.041208585209034865) + | > avg_loss_disc_real_3: 0.21992435639208935 (-0.004234341230798272) + | > avg_loss_disc_real_4: 0.21660929918289185 (-0.016201250730676853) + | > avg_loss_disc_real_5: 0.231137197702489 (-0.014091485675345072) + | > avg_loss_0: 2.7140709187122103 (+0.09481511217482641) + | > avg_loss_gen: 1.716788629268078 (-0.4565734178461931) + | > avg_loss_kl: 2.614422739820277 (+0.017227119587836626) + | > avg_loss_feat: 4.151573627553087 (-0.10121295807209396) + | > avg_loss_mel: 20.453790015362678 (-0.036473822086417584) + | > avg_loss_duration: 1.3704361814133665 (+0.0043331004203635) + | > avg_loss_1: 30.307011218781167 (-0.5727000135056528) + + + > EPOCH: 57/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 11:54:22)  + + --> TIME: 2024-06-25 11:54:26 -- STEP: 4/591 -- GLOBAL_STEP: 33100 + | > loss_disc: 2.6918673515319824 (2.7286534309387207) + | > loss_disc_real_0: 0.1689288169145584 (0.23231101781129837) + | > loss_disc_real_1: 0.1444113552570343 (0.22159092128276825) + | > loss_disc_real_2: 0.2249617874622345 (0.2501833438873291) + | > loss_disc_real_3: 0.2084224671125412 (0.23721175268292427) + | > loss_disc_real_4: 0.23436179757118225 (0.22729134932160378) + | > loss_disc_real_5: 0.28056809306144714 (0.26081886142492294) + | > loss_0: 2.6918673515319824 (2.7286534309387207) + | > grad_norm_0: tensor(12.7996, device='cuda:0') (tensor(24.2967, device='cuda:0')) + | > loss_gen: 1.9104163646697998 (2.0308324098587036) + | > loss_kl: 2.663064956665039 (2.6892297863960266) + | > loss_feat: 4.3474812507629395 (4.557285189628601) + | > loss_mel: 20.248653411865234 (20.645507335662842) + | > loss_duration: 1.347414255142212 (1.3376303017139435) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.51702880859375 (31.260483264923096) + | > grad_norm_1: tensor(725.7853, device='cuda:0') (tensor(859.6757, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.754 (0.759707510471344) + | > loader_time: 0.0089 (0.007680833339691162) + + + --> TIME: 2024-06-25 11:54:46 -- STEP: 29/591 -- GLOBAL_STEP: 33125 + | > loss_disc: 2.599093437194824 (2.657658239890789) + | > loss_disc_real_0: 0.18459883332252502 (0.2022252332033782) + | > loss_disc_real_1: 0.2043914496898651 (0.2237842339893867) + | > loss_disc_real_2: 0.22981546819210052 (0.23450253801099186) + | > loss_disc_real_3: 0.21437229216098785 (0.23317393352245464) + | > loss_disc_real_4: 0.21068434417247772 (0.22274676890208803) + | > loss_disc_real_5: 0.256873220205307 (0.24159061035205578) + | > loss_0: 2.599093437194824 (2.657658239890789) + | > grad_norm_0: tensor(7.9292, device='cuda:0') (tensor(18.9717, device='cuda:0')) + | > loss_gen: 1.9659342765808105 (2.050574487653272) + | > loss_kl: 2.5495853424072266 (2.6276210176533676) + | > loss_feat: 4.283472061157227 (4.502362974758806) + | > loss_mel: 20.215970993041992 (20.514610685151208) + | > loss_duration: 1.3282954692840576 (1.3105924376125992) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.343257904052734 (31.005761245201374) + | > grad_norm_1: tensor(117.9783, device='cuda:0') (tensor(862.5268, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.787 (0.7873977792674097) + | > loader_time: 0.0093 (0.008688014129112506) + + + --> TIME: 2024-06-25 11:55:07 -- STEP: 54/591 -- GLOBAL_STEP: 33150 + | > loss_disc: 2.7288320064544678 (2.644072267744276) + | > loss_disc_real_0: 0.22302430868148804 (0.20153189795436682) + | > loss_disc_real_1: 0.3066219985485077 (0.22401730743823228) + | > loss_disc_real_2: 0.2284442037343979 (0.2325056297359643) + | > loss_disc_real_3: 0.19713911414146423 (0.23224000605168166) + | > loss_disc_real_4: 0.3130340576171875 (0.21610727161169052) + | > loss_disc_real_5: 0.24967099726200104 (0.23985711623121192) + | > loss_0: 2.7288320064544678 (2.644072267744276) + | > grad_norm_0: tensor(37.3588, device='cuda:0') (tensor(22.0621, device='cuda:0')) + | > loss_gen: 2.1637566089630127 (2.085005623322946) + | > loss_kl: 2.585339307785034 (2.5908484856287637) + | > loss_feat: 4.399230003356934 (4.544766982396443) + | > loss_mel: 20.551918029785156 (20.54081192723028) + | > loss_duration: 1.3007320165634155 (1.3233129293830306) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.0009765625 (31.084745795638472) + | > grad_norm_1: tensor(912.1629, device='cuda:0') (tensor(976.8110, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.8194 (0.7944520667747215) + | > loader_time: 0.011 (0.009385700579042788) + + + --> TIME: 2024-06-25 11:55:28 -- STEP: 79/591 -- GLOBAL_STEP: 33175 + | > loss_disc: 2.699288845062256 (2.6465779467474055) + | > loss_disc_real_0: 0.27961069345474243 (0.20038934287768376) + | > loss_disc_real_1: 0.2574528157711029 (0.22389265743991996) + | > loss_disc_real_2: 0.23099607229232788 (0.23214306050463568) + | > loss_disc_real_3: 0.2626361548900604 (0.2333790275115001) + | > loss_disc_real_4: 0.21978649497032166 (0.21813958015622972) + | > loss_disc_real_5: 0.24741877615451813 (0.239874449900434) + | > loss_0: 2.699288845062256 (2.6465779467474055) + | > grad_norm_0: tensor(18.6121, device='cuda:0') (tensor(21.3944, device='cuda:0')) + | > loss_gen: 1.929426670074463 (2.070798262765136) + | > loss_kl: 2.5769405364990234 (2.568315762507764) + | > loss_feat: 4.25587272644043 (4.494556366642819) + | > loss_mel: 20.318471908569336 (20.60119443000118) + | > loss_duration: 1.3083600997924805 (1.3303203175339517) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.38907241821289 (31.065185064001927) + | > grad_norm_1: tensor(449.2939, device='cuda:0') (tensor(916.4070, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.8215 (0.8025414309924164) + | > loader_time: 0.0118 (0.009759839577010916) + + + --> TIME: 2024-06-25 11:55:49 -- STEP: 104/591 -- GLOBAL_STEP: 33200 + | > loss_disc: 2.547067165374756 (2.6484016615610857) + | > loss_disc_real_0: 0.1986856758594513 (0.2002766323634065) + | > loss_disc_real_1: 0.1862660050392151 (0.22346870028055632) + | > loss_disc_real_2: 0.22609715163707733 (0.2321629421069072) + | > loss_disc_real_3: 0.22240762412548065 (0.23324863985180855) + | > loss_disc_real_4: 0.22158858180046082 (0.21878629578993872) + | > loss_disc_real_5: 0.24967078864574432 (0.23971432748322302) + | > loss_0: 2.547067165374756 (2.6484016615610857) + | > grad_norm_0: tensor(18.1806, device='cuda:0') (tensor(21.2516, device='cuda:0')) + | > loss_gen: 2.2165040969848633 (2.0645436197519302) + | > loss_kl: 2.477571964263916 (2.554257677151607) + | > loss_feat: 4.69037389755249 (4.489145966676565) + | > loss_mel: 20.945026397705078 (20.621423574594363) + | > loss_duration: 1.3210991621017456 (1.3365903519667106) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.65057373046875 (31.065961085833035) + | > grad_norm_1: tensor(1200.9692, device='cuda:0') (tensor(922.8975, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.8266 (0.8089938553480003) + | > loader_time: 0.0107 (0.010020792484283447) + + + --> TIME: 2024-06-25 11:56:10 -- STEP: 129/591 -- GLOBAL_STEP: 33225 + | > loss_disc: 2.6096584796905518 (2.638480125471603) + | > loss_disc_real_0: 0.25973278284072876 (0.1997153600172479) + | > loss_disc_real_1: 0.2578413486480713 (0.2228448946577634) + | > loss_disc_real_2: 0.2374076545238495 (0.23193653701811798) + | > loss_disc_real_3: 0.23158511519432068 (0.23338482846585354) + | > loss_disc_real_4: 0.17337016761302948 (0.21528455303158872) + | > loss_disc_real_5: 0.24395334720611572 (0.23910923048045285) + | > loss_0: 2.6096584796905518 (2.638480125471603) + | > grad_norm_0: tensor(26.9474, device='cuda:0') (tensor(21.7336, device='cuda:0')) + | > loss_gen: 2.0082895755767822 (2.0715131611787077) + | > loss_kl: 2.6494674682617188 (2.546640649322392) + | > loss_feat: 4.466146945953369 (4.508087465005327) + | > loss_mel: 20.426008224487305 (20.618469090424778) + | > loss_duration: 1.3071222305297852 (1.337640850118888) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.857032775878906 (31.08235109314438) + | > grad_norm_1: tensor(485.4862, device='cuda:0') (tensor(965.8056, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.8412 (0.8161132742268171) + | > loader_time: 0.0128 (0.01030449349750844) + + + --> TIME: 2024-06-25 11:56:32 -- STEP: 154/591 -- GLOBAL_STEP: 33250 + | > loss_disc: 2.6349880695343018 (2.6386336924193743) + | > loss_disc_real_0: 0.1700892597436905 (0.1989360236405552) + | > loss_disc_real_1: 0.2392285019159317 (0.2232975866887476) + | > loss_disc_real_2: 0.2208392471075058 (0.23182993788610806) + | > loss_disc_real_3: 0.2421501725912094 (0.23364988153243993) + | > loss_disc_real_4: 0.24386152625083923 (0.21487504553485226) + | > loss_disc_real_5: 0.25376200675964355 (0.2388399586855591) + | > loss_0: 2.6349880695343018 (2.6386336924193743) + | > grad_norm_0: tensor(18.3516, device='cuda:0') (tensor(22.1947, device='cuda:0')) + | > loss_gen: 2.10715913772583 (2.077263560387996) + | > loss_kl: 2.506375551223755 (2.544226250091157) + | > loss_feat: 4.257490634918213 (4.511791786590178) + | > loss_mel: 19.84624481201172 (20.611787127209965) + | > loss_duration: 1.3137073516845703 (1.3364953561262647) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.030977249145508 (31.081563974355724) + | > grad_norm_1: tensor(892.3099, device='cuda:0') (tensor(979.4635, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.8435 (0.8217422544182118) + | > loader_time: 0.0112 (0.01059571643928429) + + + --> TIME: 2024-06-25 11:56:54 -- STEP: 179/591 -- GLOBAL_STEP: 33275 + | > loss_disc: 2.6056666374206543 (2.6395148325232825) + | > loss_disc_real_0: 0.18179114162921906 (0.19877975067779338) + | > loss_disc_real_1: 0.2522028088569641 (0.2231161604380474) + | > loss_disc_real_2: 0.23960000276565552 (0.23193844428608537) + | > loss_disc_real_3: 0.22976292669773102 (0.23324849502334383) + | > loss_disc_real_4: 0.23648111522197723 (0.21653933764835975) + | > loss_disc_real_5: 0.2532006502151489 (0.238710392453817) + | > loss_0: 2.6056666374206543 (2.6395148325232825) + | > grad_norm_0: tensor(19.9189, device='cuda:0') (tensor(21.3113, device='cuda:0')) + | > loss_gen: 2.124452590942383 (2.06996575147746) + | > loss_kl: 2.5470054149627686 (2.538216420392085) + | > loss_feat: 4.2913618087768555 (4.499895937615933) + | > loss_mel: 20.27054786682129 (20.594827960989335) + | > loss_duration: 1.3574352264404297 (1.3348986170145387) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.590803146362305 (31.037804630215607) + | > grad_norm_1: tensor(1082.3357, device='cuda:0') (tensor(951.2042, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.863 (0.8274499091356157) + | > loader_time: 0.0117 (0.010791621394663551) + + + --> TIME: 2024-06-25 11:57:16 -- STEP: 204/591 -- GLOBAL_STEP: 33300 + | > loss_disc: 2.6457598209381104 (2.642632486773472) + | > loss_disc_real_0: 0.24633321166038513 (0.1994757401008232) + | > loss_disc_real_1: 0.1837483048439026 (0.2233149762101033) + | > loss_disc_real_2: 0.2235623449087143 (0.23278598947560086) + | > loss_disc_real_3: 0.17770858108997345 (0.23344310935513646) + | > loss_disc_real_4: 0.21071431040763855 (0.21664380720433066) + | > loss_disc_real_5: 0.236282616853714 (0.2384939718188024) + | > loss_0: 2.6457598209381104 (2.642632486773472) + | > grad_norm_0: tensor(17.4146, device='cuda:0') (tensor(21.1466, device='cuda:0')) + | > loss_gen: 2.0716304779052734 (2.0655716231056283) + | > loss_kl: 2.563143253326416 (2.5357933804100656) + | > loss_feat: 4.292809009552002 (4.47996898725921) + | > loss_mel: 20.511938095092773 (20.596157513412773) + | > loss_duration: 1.3119747638702393 (1.3338424499128376) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.751495361328125 (31.011333905014336) + | > grad_norm_1: tensor(1057.9664, device='cuda:0') (tensor(946.4094, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.8865 (0.8327780202323313) + | > loader_time: 0.0138 (0.011072293216106938) + + + --> TIME: 2024-06-25 11:57:39 -- STEP: 229/591 -- GLOBAL_STEP: 33325 + | > loss_disc: 2.6417441368103027 (2.6399137463632107) + | > loss_disc_real_0: 0.25152209401130676 (0.19993456495380818) + | > loss_disc_real_1: 0.19581051170825958 (0.22309762668921954) + | > loss_disc_real_2: 0.2099636197090149 (0.2327808903034077) + | > loss_disc_real_3: 0.2698718011379242 (0.23337878898501918) + | > loss_disc_real_4: 0.24635937809944153 (0.2150394574913916) + | > loss_disc_real_5: 0.2355835735797882 (0.23826911850267102) + | > loss_0: 2.6417441368103027 (2.6399137463632107) + | > grad_norm_0: tensor(22.2523, device='cuda:0') (tensor(21.8281, device='cuda:0')) + | > loss_gen: 2.0588266849517822 (2.07329231318428) + | > loss_kl: 2.4985885620117188 (2.5342619908428627) + | > loss_feat: 4.530806541442871 (4.489049823003047) + | > loss_mel: 20.164424896240234 (20.57485900279216) + | > loss_duration: 1.3686439990997314 (1.3332953937189025) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.621292114257812 (31.0047585183356) + | > grad_norm_1: tensor(1048.9974, device='cuda:0') (tensor(970.3864, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.9036 (0.8378613973809119) + | > loader_time: 0.0132 (0.011348265227272) + + + --> TIME: 2024-06-25 11:58:02 -- STEP: 254/591 -- GLOBAL_STEP: 33350 + | > loss_disc: 2.6935417652130127 (2.6372750884904637) + | > loss_disc_real_0: 0.22789981961250305 (0.19965756608275917) + | > loss_disc_real_1: 0.20528246462345123 (0.22322330864395679) + | > loss_disc_real_2: 0.21798576414585114 (0.2329930735854652) + | > loss_disc_real_3: 0.24305309355258942 (0.23319132907653417) + | > loss_disc_real_4: 0.21741780638694763 (0.2139132156616121) + | > loss_disc_real_5: 0.2419818490743637 (0.23828576534517168) + | > loss_0: 2.6935417652130127 (2.6372750884904637) + | > grad_norm_0: tensor(19.8033, device='cuda:0') (tensor(21.8548, device='cuda:0')) + | > loss_gen: 1.9145811796188354 (2.0753562225131543) + | > loss_kl: 2.3194684982299805 (2.5303307152169916) + | > loss_feat: 4.236924648284912 (4.49302179137553) + | > loss_mel: 19.615402221679688 (20.559129444632944) + | > loss_duration: 1.3029115200042725 (1.333313177889726) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.3892879486084 (30.991151314082106) + | > grad_norm_1: tensor(851.8849, device='cuda:0') (tensor(990.2994, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.9214 (0.843828472565478) + | > loader_time: 0.0153 (0.011665468140849917) + + + --> TIME: 2024-06-25 11:58:25 -- STEP: 279/591 -- GLOBAL_STEP: 33375 + | > loss_disc: 2.60978102684021 (2.6369245265974364) + | > loss_disc_real_0: 0.18816441297531128 (0.1993738350902407) + | > loss_disc_real_1: 0.22027373313903809 (0.2231056847773145) + | > loss_disc_real_2: 0.2186376005411148 (0.2327169902008494) + | > loss_disc_real_3: 0.24362753331661224 (0.23308156072879777) + | > loss_disc_real_4: 0.18528883159160614 (0.2142007298580634) + | > loss_disc_real_5: 0.22169660031795502 (0.2381501919372962) + | > loss_0: 2.60978102684021 (2.6369245265974364) + | > grad_norm_0: tensor(18.3372, device='cuda:0') (tensor(21.7213, device='cuda:0')) + | > loss_gen: 2.0793232917785645 (2.0720765641940537) + | > loss_kl: 2.5548105239868164 (2.5280444271675586) + | > loss_feat: 4.563024044036865 (4.483879726847442) + | > loss_mel: 21.22643280029297 (20.541116693968412) + | > loss_duration: 1.3280235528945923 (1.3336441717694736) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.75161361694336 (30.95876153651959) + | > grad_norm_1: tensor(1058.8958, device='cuda:0') (tensor(986.1301, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.9395 (0.8497789299188975) + | > loader_time: 0.0161 (0.011934573505087137) + + + --> TIME: 2024-06-25 11:58:48 -- STEP: 304/591 -- GLOBAL_STEP: 33400 + | > loss_disc: 2.6856441497802734 (2.6363518951754803) + | > loss_disc_real_0: 0.22779621183872223 (0.19950045478579248) + | > loss_disc_real_1: 0.22511553764343262 (0.22269953282451943) + | > loss_disc_real_2: 0.2011229246854782 (0.232839176970485) + | > loss_disc_real_3: 0.22166793048381805 (0.2330616728745793) + | > loss_disc_real_4: 0.15145951509475708 (0.21371871154559277) + | > loss_disc_real_5: 0.2047232836484909 (0.238196528075557) + | > loss_0: 2.6856441497802734 (2.6363518951754803) + | > grad_norm_0: tensor(21.6387, device='cuda:0') (tensor(22.0733, device='cuda:0')) + | > loss_gen: 2.321782112121582 (2.0761496981507848) + | > loss_kl: 2.463426113128662 (2.526329476582378) + | > loss_feat: 4.718037128448486 (4.487387409335689) + | > loss_mel: 21.45423126220703 (20.54535439139918) + | > loss_duration: 1.363348126411438 (1.334285099647547) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.32082748413086 (30.969506050411024) + | > grad_norm_1: tensor(1479.2234, device='cuda:0') (tensor(994.0167, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.9285 (0.8556127861926427) + | > loader_time: 0.0166 (0.012194409966468808) + + + --> TIME: 2024-06-25 11:59:12 -- STEP: 329/591 -- GLOBAL_STEP: 33425 + | > loss_disc: 2.707390785217285 (2.637571897187854) + | > loss_disc_real_0: 0.16641086339950562 (0.19969616508773758) + | > loss_disc_real_1: 0.2137530893087387 (0.22329450944455562) + | > loss_disc_real_2: 0.2345377802848816 (0.23277565456451255) + | > loss_disc_real_3: 0.2620522677898407 (0.23315679189100816) + | > loss_disc_real_4: 0.2156522572040558 (0.21398380217581164) + | > loss_disc_real_5: 0.2579812705516815 (0.23819489527980608) + | > loss_0: 2.707390785217285 (2.637571897187854) + | > grad_norm_0: tensor(23.4948, device='cuda:0') (tensor(22.2086, device='cuda:0')) + | > loss_gen: 1.912280797958374 (2.073931638593005) + | > loss_kl: 2.4995014667510986 (2.525332754506169) + | > loss_feat: 4.182466983795166 (4.482386731086895) + | > loss_mel: 20.345596313476562 (20.550928237590377) + | > loss_duration: 1.3800997734069824 (1.3350584746131775) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.3199462890625 (30.967637801242816) + | > grad_norm_1: tensor(1291.0924, device='cuda:0') (tensor(991.6111, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.9179 (0.860546673684859) + | > loader_time: 0.0154 (0.012497787417611815) + + + --> TIME: 2024-06-25 11:59:36 -- STEP: 354/591 -- GLOBAL_STEP: 33450 + | > loss_disc: 2.5628461837768555 (2.6370462717982996) + | > loss_disc_real_0: 0.18587522208690643 (0.19949661203696903) + | > loss_disc_real_1: 0.19018323719501495 (0.22318956714733845) + | > loss_disc_real_2: 0.2168322056531906 (0.23245006075686653) + | > loss_disc_real_3: 0.24430808424949646 (0.23309170850421077) + | > loss_disc_real_4: 0.2083355188369751 (0.21435083650936507) + | > loss_disc_real_5: 0.23979495465755463 (0.23818477243185043) + | > loss_0: 2.5628461837768555 (2.6370462717982996) + | > grad_norm_0: tensor(18.1286, device='cuda:0') (tensor(22.0916, device='cuda:0')) + | > loss_gen: 2.169999361038208 (2.0709671169351043) + | > loss_kl: 2.5475430488586426 (2.521341071963986) + | > loss_feat: 4.399862766265869 (4.4716446130289205) + | > loss_mel: 20.821205139160156 (20.540222189520698) + | > loss_duration: 1.3666377067565918 (1.3364094512610785) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.30525016784668 (30.94058442519883) + | > grad_norm_1: tensor(1189.9492, device='cuda:0') (tensor(986.4859, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.9483 (0.8658385936823265) + | > loader_time: 0.018 (0.012770322083079882) + + + --> TIME: 2024-06-25 12:00:00 -- STEP: 379/591 -- GLOBAL_STEP: 33475 + | > loss_disc: 2.7026047706604004 (2.635881028892495) + | > loss_disc_real_0: 0.17146548628807068 (0.19928556018187063) + | > loss_disc_real_1: 0.2262469381093979 (0.2230650929237733) + | > loss_disc_real_2: 0.2105933576822281 (0.23238691549974255) + | > loss_disc_real_3: 0.2437806874513626 (0.23295785840551583) + | > loss_disc_real_4: 0.2544260025024414 (0.21398729357209864) + | > loss_disc_real_5: 0.2304258495569229 (0.23818670626051508) + | > loss_0: 2.7026047706604004 (2.635881028892495) + | > grad_norm_0: tensor(5.6571, device='cuda:0') (tensor(22.3596, device='cuda:0')) + | > loss_gen: 2.042073965072632 (2.0738189965879683) + | > loss_kl: 2.4624438285827637 (2.5179566456334275) + | > loss_feat: 4.262059688568115 (4.471342917797118) + | > loss_mel: 20.500717163085938 (20.52691273953479) + | > loss_duration: 1.3549635410308838 (1.3379403440807613) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.622257232666016 (30.92797164111779) + | > grad_norm_1: tensor(252.3475, device='cuda:0') (tensor(993.9189, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.9394 (0.8712283294244929) + | > loader_time: 0.0152 (0.013039344847988642) + + + --> TIME: 2024-06-25 12:00:25 -- STEP: 404/591 -- GLOBAL_STEP: 33500 + | > loss_disc: 2.656251907348633 (2.638020549670302) + | > loss_disc_real_0: 0.217421755194664 (0.19927739431288574) + | > loss_disc_real_1: 0.24627520143985748 (0.22321273463934954) + | > loss_disc_real_2: 0.20983155071735382 (0.23230109157243578) + | > loss_disc_real_3: 0.18931443989276886 (0.2331800667366179) + | > loss_disc_real_4: 0.2389107495546341 (0.21521230148944517) + | > loss_disc_real_5: 0.2546609044075012 (0.23820528144588565) + | > loss_0: 2.656251907348633 (2.638020549670302) + | > grad_norm_0: tensor(11.8944, device='cuda:0') (tensor(21.8044, device='cuda:0')) + | > loss_gen: 1.9228688478469849 (2.0680856456851004) + | > loss_kl: 2.5221097469329834 (2.5167654723224087) + | > loss_feat: 4.243462562561035 (4.454680379074398) + | > loss_mel: 20.601810455322266 (20.517147503276853) + | > loss_duration: 1.3414050340652466 (1.3390873655824378) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.631656646728516 (30.895766366826425) + | > grad_norm_1: tensor(607.3329, device='cuda:0') (tensor(964.7858, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.9882 (0.8776210871073277) + | > loader_time: 0.0169 (0.013314631315741208) + + + --> TIME: 2024-06-25 12:00:50 -- STEP: 429/591 -- GLOBAL_STEP: 33525 + | > loss_disc: 2.623217821121216 (2.638586197143944) + | > loss_disc_real_0: 0.27016380429267883 (0.19894443843122964) + | > loss_disc_real_1: 0.23430360853672028 (0.22314091926410204) + | > loss_disc_real_2: 0.23598067462444305 (0.23244468086249345) + | > loss_disc_real_3: 0.198171928524971 (0.2332353600949952) + | > loss_disc_real_4: 0.2567717730998993 (0.21587268445458438) + | > loss_disc_real_5: 0.24357691407203674 (0.23817988792499462) + | > loss_0: 2.623217821121216 (2.638586197143944) + | > grad_norm_0: tensor(28.5341, device='cuda:0') (tensor(21.5796, device='cuda:0')) + | > loss_gen: 2.0024027824401855 (2.064445773482599) + | > loss_kl: 2.4372971057891846 (2.5136446085843183) + | > loss_feat: 3.993652105331421 (4.442123301379331) + | > loss_mel: 19.95009422302246 (20.50322746063445) + | > loss_duration: 1.3540403842926025 (1.339930557704472) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.737485885620117 (30.86337170456395) + | > grad_norm_1: tensor(1094.4982, device='cuda:0') (tensor(952.3521, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 0.9766 (0.8837152272115496) + | > loader_time: 0.0162 (0.013580900805813449) + + + --> TIME: 2024-06-25 12:01:16 -- STEP: 454/591 -- GLOBAL_STEP: 33550 + | > loss_disc: 2.509331703186035 (2.6380288648185206) + | > loss_disc_real_0: 0.20250795781612396 (0.19895294543339292) + | > loss_disc_real_1: 0.20347736775875092 (0.2232864237829452) + | > loss_disc_real_2: 0.24116653203964233 (0.2323940835466469) + | > loss_disc_real_3: 0.2778984308242798 (0.23329876074444356) + | > loss_disc_real_4: 0.15946732461452484 (0.21540387188154164) + | > loss_disc_real_5: 0.22483059763908386 (0.23816979591148016) + | > loss_0: 2.509331703186035 (2.6380288648185206) + | > grad_norm_0: tensor(23.8483, device='cuda:0') (tensor(21.8558, device='cuda:0')) + | > loss_gen: 2.307035446166992 (2.0667323223294645) + | > loss_kl: 2.5114336013793945 (2.511969378341138) + | > loss_feat: 4.8109893798828125 (4.438449674240821) + | > loss_mel: 19.995990753173828 (20.49375818899549) + | > loss_duration: 1.3708980083465576 (1.3409290639314357) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.99634552001953 (30.851838637028496) + | > grad_norm_1: tensor(1315.2438, device='cuda:0') (tensor(956.3832, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 1.0005 (0.8903390379204096) + | > loader_time: 0.0225 (0.013876882418661916) + + + --> TIME: 2024-06-25 12:01:42 -- STEP: 479/591 -- GLOBAL_STEP: 33575 + | > loss_disc: 2.6467318534851074 (2.6374260866567316) + | > loss_disc_real_0: 0.1314750611782074 (0.19890401571491317) + | > loss_disc_real_1: 0.2176637202501297 (0.22345067604225213) + | > loss_disc_real_2: 0.22118045389652252 (0.23229682308646982) + | > loss_disc_real_3: 0.23198272287845612 (0.2331242740838662) + | > loss_disc_real_4: 0.19680233299732208 (0.21489248995616683) + | > loss_disc_real_5: 0.21491840481758118 (0.23802603778112408) + | > loss_0: 2.6467318534851074 (2.6374260866567316) + | > grad_norm_0: tensor(19.2593, device='cuda:0') (tensor(22.1335, device='cuda:0')) + | > loss_gen: 2.0458617210388184 (2.0684396362006034) + | > loss_kl: 2.397104263305664 (2.5106486199046483) + | > loss_feat: 4.399111747741699 (4.436379471999869) + | > loss_mel: 20.421066284179688 (20.488904522953543) + | > loss_duration: 1.411192536354065 (1.342241063247394) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.674335479736328 (30.846613312563967) + | > grad_norm_1: tensor(400.6353, device='cuda:0') (tensor(954.8575, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 1.0042 (0.8968050982608673) + | > loader_time: 0.0181 (0.014208342685580005) + + + --> TIME: 2024-06-25 12:02:09 -- STEP: 504/591 -- GLOBAL_STEP: 33600 + | > loss_disc: 2.7032063007354736 (2.6380751241767197) + | > loss_disc_real_0: 0.16819101572036743 (0.19885732610488221) + | > loss_disc_real_1: 0.26028117537498474 (0.2234198869102531) + | > loss_disc_real_2: 0.2361198514699936 (0.23227837734988757) + | > loss_disc_real_3: 0.2601325213909149 (0.23322051116043613) + | > loss_disc_real_4: 0.2080656886100769 (0.2152832557875958) + | > loss_disc_real_5: 0.21025657653808594 (0.2379145599192097) + | > loss_0: 2.7032063007354736 (2.6380751241767197) + | > grad_norm_0: tensor(6.8206, device='cuda:0') (tensor(21.9267, device='cuda:0')) + | > loss_gen: 1.9238274097442627 (2.0647199090038018) + | > loss_kl: 2.553126096725464 (2.5100839081264685) + | > loss_feat: 3.9057350158691406 (4.422325586515761) + | > loss_mel: 19.93491554260254 (20.47103074240305) + | > loss_duration: 1.3772625923156738 (1.343781983568555) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.694866180419922 (30.811942123231432) + | > grad_norm_1: tensor(46.1184, device='cuda:0') (tensor(945.4894, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 1.0684 (0.904289117408177) + | > loader_time: 0.0225 (0.01452100087725927) + + + --> TIME: 2024-06-25 12:02:36 -- STEP: 529/591 -- GLOBAL_STEP: 33625 + | > loss_disc: 2.584737539291382 (2.638764162370071) + | > loss_disc_real_0: 0.1882067769765854 (0.1987512753202469) + | > loss_disc_real_1: 0.2474922239780426 (0.22348329005146747) + | > loss_disc_real_2: 0.2351199984550476 (0.23231983221433564) + | > loss_disc_real_3: 0.24375033378601074 (0.2333059712878247) + | > loss_disc_real_4: 0.22676101326942444 (0.21554632484912867) + | > loss_disc_real_5: 0.223024383187294 (0.23799844012931973) + | > loss_0: 2.584737539291382 (2.638764162370071) + | > grad_norm_0: tensor(31.8774, device='cuda:0') (tensor(21.8163, device='cuda:0')) + | > loss_gen: 2.082608699798584 (2.0627605989434534) + | > loss_kl: 2.534313201904297 (2.5075221854932828) + | > loss_feat: 4.556623935699463 (4.413570705118163) + | > loss_mel: 20.34395408630371 (20.459352785337618) + | > loss_duration: 1.3784643411636353 (1.3455459055242556) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.895963668823242 (30.788752168247967) + | > grad_norm_1: tensor(1592.9727, device='cuda:0') (tensor(943.0414, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 1.1337 (0.9124762921793022) + | > loader_time: 0.0182 (0.014817442921023738) + + + --> TIME: 2024-06-25 12:03:05 -- STEP: 554/591 -- GLOBAL_STEP: 33650 + | > loss_disc: 2.7270376682281494 (2.639977231783126) + | > loss_disc_real_0: 0.16548484563827515 (0.19895977998952574) + | > loss_disc_real_1: 0.2605988085269928 (0.22379189331608995) + | > loss_disc_real_2: 0.2474057972431183 (0.23227870862406513) + | > loss_disc_real_3: 0.2779700756072998 (0.23333666575837222) + | > loss_disc_real_4: 0.287640780210495 (0.21544609337184398) + | > loss_disc_real_5: 0.2116984874010086 (0.2380324851197026) + | > loss_0: 2.7270376682281494 (2.639977231783126) + | > grad_norm_0: tensor(19.4561, device='cuda:0') (tensor(22.0643, device='cuda:0')) + | > loss_gen: 2.0758180618286133 (2.0646469431663674) + | > loss_kl: 2.4089133739471436 (2.5048685887230016) + | > loss_feat: 4.1356201171875 (4.40589524649541) + | > loss_mel: 20.522674560546875 (20.44764937046202) + | > loss_duration: 1.3811746835708618 (1.3471890468459702) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.524198532104492 (30.770249170516802) + | > grad_norm_1: tensor(723.6838, device='cuda:0') (tensor(941.7839, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 1.0985 (0.9216123957065897) + | > loader_time: 0.0248 (0.015177046779260739) + + + --> TIME: 2024-06-25 12:03:35 -- STEP: 579/591 -- GLOBAL_STEP: 33675 + | > loss_disc: 2.6376283168792725 (2.6413476862437983) + | > loss_disc_real_0: 0.27105987071990967 (0.19944201906522116) + | > loss_disc_real_1: 0.22827886044979095 (0.22373283573382877) + | > loss_disc_real_2: 0.2172446995973587 (0.23230092690707488) + | > loss_disc_real_3: 0.21794728934764862 (0.23330552894010856) + | > loss_disc_real_4: 0.16478504240512848 (0.21570255186582468) + | > loss_disc_real_5: 0.22429215908050537 (0.23805772504143557) + | > loss_0: 2.6376283168792725 (2.6413476862437983) + | > grad_norm_0: tensor(33.6267, device='cuda:0') (tensor(22.0409, device='cuda:0')) + | > loss_gen: 1.944283366203308 (2.0613705825723305) + | > loss_kl: 2.5067262649536133 (2.5020094761988463) + | > loss_feat: 4.305508613586426 (4.391130605098085) + | > loss_mel: 20.609146118164062 (20.429319289475934) + | > loss_duration: 1.389889121055603 (1.3489096784426966) + | > amp_scaler: 128.0 (65.5474956822107) + | > loss_1: 30.755552291870117 (30.732739605022424) + | > grad_norm_1: tensor(1059.5812, device='cuda:0') (tensor(932.1372, device='cuda:0')) + | > current_lr_0: 0.00019860480168978534 + | > current_lr_1: 0.00019860480168978534 + | > step_time: 1.3598 (0.9326166464257115) + | > loader_time: 0.0253 (0.015596566834392119) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035572559275525682 (-5.059039339106114e-05) + | > avg_loss_disc: 2.700486822331205 (-0.013584096381005217) + | > avg_loss_disc_real_0: 0.1799686516218997 (+0.06639656662306886) + | > avg_loss_disc_real_1: 0.22619209169073307 (+0.025405094027519226) + | > avg_loss_disc_real_2: 0.20531763960706426 (-0.04175101727881331) + | > avg_loss_disc_real_3: 0.19663539211800757 (-0.02328896427408178) + | > avg_loss_disc_real_4: 0.2468952295628 (+0.03028593037990815) + | > avg_loss_disc_real_5: 0.24430268907800634 (+0.01316549137551734) + | > avg_loss_0: 2.700486822331205 (-0.013584096381005217) + | > avg_loss_gen: 1.8242782607991646 (+0.10748963153108648) + | > avg_loss_kl: 2.5474712595026543 (-0.06695148031762255) + | > avg_loss_feat: 4.084818139989325 (-0.06675548756376148) + | > avg_loss_mel: 20.228696254973723 (-0.22509376038895468) + | > avg_loss_duration: 1.3647909976066428 (-0.005645183806723741) + | > avg_loss_1: 30.050054753080328 (-0.2569564657008385) + + + > EPOCH: 58/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 12:03:57)  + + --> TIME: 2024-06-25 12:04:09 -- STEP: 13/591 -- GLOBAL_STEP: 33700 + | > loss_disc: 2.6307103633880615 (2.6185069267566385) + | > loss_disc_real_0: 0.28192439675331116 (0.2052203233425434) + | > loss_disc_real_1: 0.21078187227249146 (0.22231064966091743) + | > loss_disc_real_2: 0.2414279729127884 (0.22785689395207626) + | > loss_disc_real_3: 0.2543133795261383 (0.23529554559634283) + | > loss_disc_real_4: 0.23103593289852142 (0.21943649764244372) + | > loss_disc_real_5: 0.2128249704837799 (0.23572309085955986) + | > loss_0: 2.6307103633880615 (2.6185069267566385) + | > grad_norm_0: tensor(30.1369, device='cuda:0') (tensor(22.9997, device='cuda:0')) + | > loss_gen: 1.9643077850341797 (2.0932371616363525) + | > loss_kl: 2.6251296997070312 (2.614674769915067) + | > loss_feat: 4.639250755310059 (4.7086700292734) + | > loss_mel: 20.734336853027344 (20.696608763474686) + | > loss_duration: 1.3182240724563599 (1.3125738639097948) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 31.28125 (31.42576481745793) + | > grad_norm_1: tensor(1203.7075, device='cuda:0') (tensor(1016.0268, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.777 (0.772557442004864) + | > loader_time: 0.0091 (0.00872490956233098) + + + --> TIME: 2024-06-25 12:04:29 -- STEP: 38/591 -- GLOBAL_STEP: 33725 + | > loss_disc: 2.602013111114502 (2.6341059458883187) + | > loss_disc_real_0: 0.1608191579580307 (0.19786014407873154) + | > loss_disc_real_1: 0.21203109622001648 (0.22178768445002406) + | > loss_disc_real_2: 0.2215995192527771 (0.23201368082510798) + | > loss_disc_real_3: 0.2207646369934082 (0.23392591272529803) + | > loss_disc_real_4: 0.20424729585647583 (0.22008886188268661) + | > loss_disc_real_5: 0.20143167674541473 (0.2367735858026304) + | > loss_0: 2.602013111114502 (2.6341059458883187) + | > grad_norm_0: tensor(24.9495, device='cuda:0') (tensor(20.8291, device='cuda:0')) + | > loss_gen: 2.046640634536743 (2.059792139028248) + | > loss_kl: 2.533841848373413 (2.6004255257154765) + | > loss_feat: 4.360983371734619 (4.543278054187172) + | > loss_mel: 20.4659366607666 (20.553600060312373) + | > loss_duration: 1.3107677698135376 (1.3157803949556852) + | > amp_scaler: 128.0 (128.0) + | > loss_1: 30.718170166015625 (31.07287627772281) + | > grad_norm_1: tensor(985.7160, device='cuda:0') (tensor(969.5007, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.817 (0.7846401678888423) + | > loader_time: 0.0103 (0.009334087371826172) + + + --> TIME: 2024-06-25 12:04:49 -- STEP: 63/591 -- GLOBAL_STEP: 33750 + | > loss_disc: 2.5878541469573975 (2.629524174190703) + | > loss_disc_real_0: 0.20751437544822693 (0.19856934284880048) + | > loss_disc_real_1: 0.19437472522258759 (0.22496491954440162) + | > loss_disc_real_2: 0.20306891202926636 (0.23205961736421737) + | > loss_disc_real_3: 0.2008790522813797 (0.2339331124510084) + | > loss_disc_real_4: 0.10211926698684692 (0.21148796261303007) + | > loss_disc_real_5: 0.22705674171447754 (0.23747323666300094) + | > loss_0: 2.5878541469573975 (2.629524174190703) + | > grad_norm_0: tensor(25.6877, device='cuda:0') (tensor(23.9681, device='cuda:0')) + | > loss_gen: 2.076873779296875 (2.118570499949986) + | > loss_kl: 2.5304291248321533 (2.5784764857519242) + | > loss_feat: 4.842722415924072 (4.644012572273375) + | > loss_mel: 20.85800552368164 (20.546295559595503) + | > loss_duration: 1.35982084274292 (1.3240259961476402) + | > amp_scaler: 64.0 (110.73015873015873) + | > loss_1: 31.6678524017334 (31.211381185622443) + | > grad_norm_1: tensor(1322.8896, device='cuda:0') (tensor(994.6405, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.828 (0.7943579923538936) + | > loader_time: 0.0113 (0.009689448371766106) + + + --> TIME: 2024-06-25 12:05:11 -- STEP: 88/591 -- GLOBAL_STEP: 33775 + | > loss_disc: 2.598454713821411 (2.639206233349714) + | > loss_disc_real_0: 0.16471640765666962 (0.19894896252927455) + | > loss_disc_real_1: 0.18443149328231812 (0.22488352639431303) + | > loss_disc_real_2: 0.21729114651679993 (0.23226093551651997) + | > loss_disc_real_3: 0.24556845426559448 (0.23358371396633712) + | > loss_disc_real_4: 0.2533864676952362 (0.21585681043903937) + | > loss_disc_real_5: 0.20729763805866241 (0.23771957138722594) + | > loss_0: 2.598454713821411 (2.639206233349714) + | > grad_norm_0: tensor(10.8905, device='cuda:0') (tensor(23.2415, device='cuda:0')) + | > loss_gen: 2.1249430179595947 (2.121185763315722) + | > loss_kl: 2.5672008991241455 (2.566795869307085) + | > loss_feat: 4.6499714851379395 (4.6217501380226835) + | > loss_mel: 20.980518341064453 (20.608079953627147) + | > loss_duration: 1.3967139720916748 (1.3312181004069068) + | > amp_scaler: 64.0 (97.45454545454548) + | > loss_1: 31.71934700012207 (31.249029766429555) + | > grad_norm_1: tensor(700.4507, device='cuda:0') (tensor(873.2031, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.8296 (0.8056519302454862) + | > loader_time: 0.0116 (0.010023225437511099) + + + --> TIME: 2024-06-25 12:05:32 -- STEP: 113/591 -- GLOBAL_STEP: 33800 + | > loss_disc: 2.6596081256866455 (2.645123355156553) + | > loss_disc_real_0: 0.16787445545196533 (0.19700839014443677) + | > loss_disc_real_1: 0.195599764585495 (0.22465036413838377) + | > loss_disc_real_2: 0.23084132373332977 (0.23341302679175824) + | > loss_disc_real_3: 0.24689246714115143 (0.23367181262083814) + | > loss_disc_real_4: 0.23873120546340942 (0.2202487067706817) + | > loss_disc_real_5: 0.2525796592235565 (0.23781498106180038) + | > loss_0: 2.6596081256866455 (2.645123355156553) + | > grad_norm_0: tensor(5.8939, device='cuda:0') (tensor(20.2119, device='cuda:0')) + | > loss_gen: 2.0393028259277344 (2.089177592665749) + | > loss_kl: 2.625404119491577 (2.55571847257361) + | > loss_feat: 4.550529956817627 (4.552415254896725) + | > loss_mel: 20.5086669921875 (20.568545991340567) + | > loss_duration: 1.3219599723815918 (1.334364561908013) + | > amp_scaler: 64.0 (90.05309734513276) + | > loss_1: 31.04586410522461 (31.100221768944664) + | > grad_norm_1: tensor(119.9136, device='cuda:0') (tensor(739.4734, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.8332 (0.8119600198965157) + | > loader_time: 0.0108 (0.010345925272038554) + + + --> TIME: 2024-06-25 12:05:53 -- STEP: 138/591 -- GLOBAL_STEP: 33825 + | > loss_disc: 2.6663289070129395 (2.6490646220635683) + | > loss_disc_real_0: 0.13256242871284485 (0.1982584009891835) + | > loss_disc_real_1: 0.2255079746246338 (0.2239462503272554) + | > loss_disc_real_2: 0.2291763573884964 (0.23299317580202353) + | > loss_disc_real_3: 0.22230473160743713 (0.23313880754985672) + | > loss_disc_real_4: 0.22975099086761475 (0.22324933225046034) + | > loss_disc_real_5: 0.23683667182922363 (0.23794313584980759) + | > loss_0: 2.6663289070129395 (2.6490646220635683) + | > grad_norm_0: tensor(22.7477, device='cuda:0') (tensor(19.0755, device='cuda:0')) + | > loss_gen: 2.0371034145355225 (2.0707376383352973) + | > loss_kl: 2.443481206893921 (2.5427832482517627) + | > loss_feat: 4.289130687713623 (4.512710655944938) + | > loss_mel: 19.986265182495117 (20.545591589333352) + | > loss_duration: 1.3539345264434814 (1.3355755615925444) + | > amp_scaler: 64.0 (85.33333333333333) + | > loss_1: 30.109914779663086 (31.00739866063215) + | > grad_norm_1: tensor(724.3043, device='cuda:0') (tensor(646.7908, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.8392 (0.8178078609964125) + | > loader_time: 0.0114 (0.010596971580947656) + + + --> TIME: 2024-06-25 12:06:15 -- STEP: 163/591 -- GLOBAL_STEP: 33850 + | > loss_disc: 2.6002542972564697 (2.6475338848090604) + | > loss_disc_real_0: 0.17126043140888214 (0.1986375755967538) + | > loss_disc_real_1: 0.2442987561225891 (0.22350720055629872) + | > loss_disc_real_2: 0.2415187805891037 (0.23250098636179614) + | > loss_disc_real_3: 0.24479256570339203 (0.2329085316577572) + | > loss_disc_real_4: 0.22353237867355347 (0.22290715470087308) + | > loss_disc_real_5: 0.24965335428714752 (0.23796092138334285) + | > loss_0: 2.6002542972564697 (2.6475338848090604) + | > grad_norm_0: tensor(14.8559, device='cuda:0') (tensor(18.8290, device='cuda:0')) + | > loss_gen: 1.9868228435516357 (2.06349944190745) + | > loss_kl: 2.496047258377075 (2.5362865178862957) + | > loss_feat: 4.726151466369629 (4.492652189512195) + | > loss_mel: 20.80131721496582 (20.513256061296534) + | > loss_duration: 1.3186945915222168 (1.3343097168975075) + | > amp_scaler: 64.0 (82.06134969325154) + | > loss_1: 31.32903289794922 (30.940003868992342) + | > grad_norm_1: tensor(770.1047, device='cuda:0') (tensor(673.9658, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.8672 (0.8238581060631874) + | > loader_time: 0.0123 (0.010829068400377152) + + + --> TIME: 2024-06-25 12:06:37 -- STEP: 188/591 -- GLOBAL_STEP: 33875 + | > loss_disc: 2.6750452518463135 (2.6464607500015416) + | > loss_disc_real_0: 0.20722638070583344 (0.19902479113574992) + | > loss_disc_real_1: 0.24934197962284088 (0.2233004075415591) + | > loss_disc_real_2: 0.2769603729248047 (0.23221371544802444) + | > loss_disc_real_3: 0.20847360789775848 (0.2323303924120487) + | > loss_disc_real_4: 0.20153285562992096 (0.2227436909253927) + | > loss_disc_real_5: 0.23412679135799408 (0.23817460001149077) + | > loss_0: 2.6750452518463135 (2.6464607500015416) + | > grad_norm_0: tensor(8.8959, device='cuda:0') (tensor(18.4153, device='cuda:0')) + | > loss_gen: 1.9433157444000244 (2.0585558928073726) + | > loss_kl: 2.3894968032836914 (2.5313010418668696) + | > loss_feat: 4.270077705383301 (4.482658411594151) + | > loss_mel: 20.478111267089844 (20.49518392441122) + | > loss_duration: 1.3342868089675903 (1.332749702194903) + | > amp_scaler: 64.0 (79.65957446808514) + | > loss_1: 30.4152889251709 (30.900448951315372) + | > grad_norm_1: tensor(643.2693, device='cuda:0') (tensor(681.7967, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.8692 (0.8293184282931876) + | > loader_time: 0.0121 (0.011051822215952772) + + + --> TIME: 2024-06-25 12:07:00 -- STEP: 213/591 -- GLOBAL_STEP: 33900 + | > loss_disc: 2.567680835723877 (2.6468794592109632) + | > loss_disc_real_0: 0.26047030091285706 (0.20086536094756194) + | > loss_disc_real_1: 0.2039954513311386 (0.223083668191668) + | > loss_disc_real_2: 0.2214469313621521 (0.23209292004365875) + | > loss_disc_real_3: 0.2506449520587921 (0.23274024733355347) + | > loss_disc_real_4: 0.2420283406972885 (0.22072385667775157) + | > loss_disc_real_5: 0.26224052906036377 (0.2383587019684169) + | > loss_0: 2.567680835723877 (2.6468794592109632) + | > grad_norm_0: tensor(26.4399, device='cuda:0') (tensor(19.7497, device='cuda:0')) + | > loss_gen: 2.0093586444854736 (2.0654014695977954) + | > loss_kl: 2.4305472373962402 (2.524622905981931) + | > loss_feat: 4.466586589813232 (4.489993648349961) + | > loss_mel: 20.873271942138672 (20.50088175025905) + | > loss_duration: 1.332794189453125 (1.3318646799230789) + | > amp_scaler: 64.0 (77.82159624413146) + | > loss_1: 31.112558364868164 (30.91276441941239) + | > grad_norm_1: tensor(1353.3175, device='cuda:0') (tensor(744.0037, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.8722 (0.8346516000273082) + | > loader_time: 0.0133 (0.01130585491377423) + + + --> TIME: 2024-06-25 12:07:22 -- STEP: 238/591 -- GLOBAL_STEP: 33925 + | > loss_disc: 2.6445465087890625 (2.6457360321734122) + | > loss_disc_real_0: 0.15428364276885986 (0.19998180039790497) + | > loss_disc_real_1: 0.20820362865924835 (0.2231221934076117) + | > loss_disc_real_2: 0.2334616482257843 (0.231938588769496) + | > loss_disc_real_3: 0.2872939109802246 (0.23286810398352245) + | > loss_disc_real_4: 0.18626001477241516 (0.22059827157053627) + | > loss_disc_real_5: 0.2413536012172699 (0.2383736683791425) + | > loss_0: 2.6445465087890625 (2.6457360321734122) + | > grad_norm_0: tensor(11.6642, device='cuda:0') (tensor(19.8589, device='cuda:0')) + | > loss_gen: 2.0874741077423096 (2.0642039199837128) + | > loss_kl: 2.545999050140381 (2.5230423061787572) + | > loss_feat: 4.319746971130371 (4.483005062872626) + | > loss_mel: 20.705638885498047 (20.50839893757797) + | > loss_duration: 1.2970690727233887 (1.3316301548180451) + | > amp_scaler: 64.0 (76.36974789915966) + | > loss_1: 30.955928802490234 (30.9102803158159) + | > grad_norm_1: tensor(794.8343, device='cuda:0') (tensor(774.5665, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.8735 (0.839128597443845) + | > loader_time: 0.0129 (0.0116087779277513) + + + --> TIME: 2024-06-25 12:07:45 -- STEP: 263/591 -- GLOBAL_STEP: 33950 + | > loss_disc: 2.607121229171753 (2.64673718753423) + | > loss_disc_real_0: 0.16687017679214478 (0.1999593741426903) + | > loss_disc_real_1: 0.2195252925157547 (0.22313446084129493) + | > loss_disc_real_2: 0.22839681804180145 (0.23190805100669426) + | > loss_disc_real_3: 0.21916703879833221 (0.2328545827376072) + | > loss_disc_real_4: 0.23169921338558197 (0.2207192074606627) + | > loss_disc_real_5: 0.22236357629299164 (0.23823654770624503) + | > loss_0: 2.607121229171753 (2.64673718753423) + | > grad_norm_0: tensor(21.0530, device='cuda:0') (tensor(19.8431, device='cuda:0')) + | > loss_gen: 2.152235984802246 (2.060390709470888) + | > loss_kl: 2.5910682678222656 (2.5204762959208318) + | > loss_feat: 4.51330041885376 (4.474318558725571) + | > loss_mel: 20.59636878967285 (20.501168914620873) + | > loss_duration: 1.3182812929153442 (1.3316285678189055) + | > amp_scaler: 64.0 (75.19391634980988) + | > loss_1: 31.171255111694336 (30.88798298853885) + | > grad_norm_1: tensor(1052.8771, device='cuda:0') (tensor(770.5861, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.9093 (0.8446329900067115) + | > loader_time: 0.015 (0.011904939499191459) + + + --> TIME: 2024-06-25 12:08:08 -- STEP: 288/591 -- GLOBAL_STEP: 33975 + | > loss_disc: 2.59692645072937 (2.645940043860011) + | > loss_disc_real_0: 0.1875164806842804 (0.19936092606642175) + | > loss_disc_real_1: 0.23308691382408142 (0.2234549660028683) + | > loss_disc_real_2: 0.24794654548168182 (0.23194929864257574) + | > loss_disc_real_3: 0.2284887731075287 (0.2329331665920715) + | > loss_disc_real_4: 0.22168920934200287 (0.22081331678459215) + | > loss_disc_real_5: 0.2280353605747223 (0.23835738618961638) + | > loss_0: 2.59692645072937 (2.645940043860011) + | > grad_norm_0: tensor(32.6765, device='cuda:0') (tensor(19.7144, device='cuda:0')) + | > loss_gen: 2.0353715419769287 (2.056954325901139) + | > loss_kl: 2.4285879135131836 (2.518712398906549) + | > loss_feat: 4.357226848602295 (4.466988879773356) + | > loss_mel: 20.413101196289062 (20.489804830816063) + | > loss_duration: 1.3659119606018066 (1.3324878116448713) + | > amp_scaler: 64.0 (74.22222222222226) + | > loss_1: 30.60019874572754 (30.864948199854958) + | > grad_norm_1: tensor(1328.4430, device='cuda:0') (tensor(777.1957, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.8943 (0.8500081209672822) + | > loader_time: 0.0131 (0.012161919640170202) + + + --> TIME: 2024-06-25 12:08:32 -- STEP: 313/591 -- GLOBAL_STEP: 34000 + | > loss_disc: 2.524401903152466 (2.643899837621866) + | > loss_disc_real_0: 0.1759684979915619 (0.1990261034557994) + | > loss_disc_real_1: 0.18032366037368774 (0.2231348953403223) + | > loss_disc_real_2: 0.22843854129314423 (0.2317891856923271) + | > loss_disc_real_3: 0.23793594539165497 (0.2330607130123785) + | > loss_disc_real_4: 0.19916225969791412 (0.2199154361701621) + | > loss_disc_real_5: 0.24212567508220673 (0.23844103239024408) + | > loss_0: 2.524401903152466 (2.643899837621866) + | > grad_norm_0: tensor(25.8671, device='cuda:0') (tensor(20.1694, device='cuda:0')) + | > loss_gen: 2.2568764686584473 (2.0603336042489513) + | > loss_kl: 2.504897117614746 (2.518593751584378) + | > loss_feat: 4.815637588500977 (4.469430593636858) + | > loss_mel: 20.315961837768555 (20.487615731577527) + | > loss_duration: 1.3249893188476562 (1.3328420163724357) + | > amp_scaler: 64.0 (73.4057507987221) + | > loss_1: 31.21836280822754 (30.86881564448055) + | > grad_norm_1: tensor(1290.9281, device='cuda:0') (tensor(802.3972, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.9164 (0.8561417774651379) + | > loader_time: 0.018 (0.012430744810987965) + + + --> TIME: 2024-06-25 12:08:56 -- STEP: 338/591 -- GLOBAL_STEP: 34025 + | > loss_disc: 2.629786252975464 (2.6435844249273908) + | > loss_disc_real_0: 0.21834853291511536 (0.19891596603146666) + | > loss_disc_real_1: 0.2262178361415863 (0.22344137031651107) + | > loss_disc_real_2: 0.24930056929588318 (0.23186560244073529) + | > loss_disc_real_3: 0.24151957035064697 (0.2329771719740693) + | > loss_disc_real_4: 0.20789755880832672 (0.21924351475383402) + | > loss_disc_real_5: 0.2298971563577652 (0.23818499542023303) + | > loss_0: 2.629786252975464 (2.6435844249273908) + | > grad_norm_0: tensor(28.4397, device='cuda:0') (tensor(20.4999, device='cuda:0')) + | > loss_gen: 1.9732518196105957 (2.06082968175764) + | > loss_kl: 2.4675564765930176 (2.5180405082081903) + | > loss_feat: 4.236300468444824 (4.464459140625232) + | > loss_mel: 21.157814025878906 (20.486127012580106) + | > loss_duration: 1.3806054592132568 (1.3339078267650484) + | > amp_scaler: 64.0 (72.71005917159769) + | > loss_1: 31.21552848815918 (30.863364095518573) + | > grad_norm_1: tensor(1094.2642, device='cuda:0') (tensor(819.9211, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.915 (0.8620473590828258) + | > loader_time: 0.0161 (0.01270148810550306) + + + --> TIME: 2024-06-25 12:09:20 -- STEP: 363/591 -- GLOBAL_STEP: 34050 + | > loss_disc: 2.609691858291626 (2.6424223444021777) + | > loss_disc_real_0: 0.12897396087646484 (0.19883630673090613) + | > loss_disc_real_1: 0.19007904827594757 (0.22320683095244995) + | > loss_disc_real_2: 0.2165134847164154 (0.23192411082342637) + | > loss_disc_real_3: 0.23639656603336334 (0.23300362417356374) + | > loss_disc_real_4: 0.21397258341312408 (0.21891037933015298) + | > loss_disc_real_5: 0.21230898797512054 (0.2380933093712678) + | > loss_0: 2.609691858291626 (2.6424223444021777) + | > grad_norm_0: tensor(35.1548, device='cuda:0') (tensor(20.9175, device='cuda:0')) + | > loss_gen: 2.196845769882202 (2.0632802973765636) + | > loss_kl: 2.522491693496704 (2.516052767593341) + | > loss_feat: 4.538088321685791 (4.461626929685107) + | > loss_mel: 20.678564071655273 (20.48563741258353) + | > loss_duration: 1.3637880086898804 (1.3353999268581707) + | > amp_scaler: 64.0 (72.11019283746562) + | > loss_1: 31.29977798461914 (30.86199728910588) + | > grad_norm_1: tensor(1755.5913, device='cuda:0') (tensor(836.9268, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.9596 (0.8682036064873057) + | > loader_time: 0.0186 (0.012979894301779671) + + + --> TIME: 2024-06-25 12:09:45 -- STEP: 388/591 -- GLOBAL_STEP: 34075 + | > loss_disc: 2.7330832481384277 (2.6421933118830028) + | > loss_disc_real_0: 0.11244532465934753 (0.1988704489332806) + | > loss_disc_real_1: 0.24926356971263885 (0.22335605619033588) + | > loss_disc_real_2: 0.22210125625133514 (0.2319660425647018) + | > loss_disc_real_3: 0.22049210965633392 (0.23311032562218995) + | > loss_disc_real_4: 0.20253801345825195 (0.2180188957832216) + | > loss_disc_real_5: 0.2463216334581375 (0.238210430656819) + | > loss_0: 2.7330832481384277 (2.6421933118830028) + | > grad_norm_0: tensor(31.5206, device='cuda:0') (tensor(21.4214, device='cuda:0')) + | > loss_gen: 2.1437675952911377 (2.064727163191922) + | > loss_kl: 2.6238045692443848 (2.513225154163911) + | > loss_feat: 4.207334518432617 (4.457309783734) + | > loss_mel: 19.90326690673828 (20.47260834015521) + | > loss_duration: 1.3276013135910034 (1.3365612008522454) + | > amp_scaler: 64.0 (71.58762886597944) + | > loss_1: 30.205774307250977 (30.844431596932953) + | > grad_norm_1: tensor(1125.1692, device='cuda:0') (tensor(859.1271, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 0.9638 (0.874764875038383) + | > loader_time: 0.0189 (0.013272279931097915) + + + --> TIME: 2024-06-25 12:10:10 -- STEP: 413/591 -- GLOBAL_STEP: 34100 + | > loss_disc: 2.5474166870117188 (2.640214156296294) + | > loss_disc_real_0: 0.2329099029302597 (0.19894894479478528) + | > loss_disc_real_1: 0.20288445055484772 (0.2232569515632948) + | > loss_disc_real_2: 0.2159813940525055 (0.23190345591696354) + | > loss_disc_real_3: 0.23820684850215912 (0.23299418010954132) + | > loss_disc_real_4: 0.16587667167186737 (0.21741060748443766) + | > loss_disc_real_5: 0.24730701744556427 (0.23845113471114318) + | > loss_0: 2.5474166870117188 (2.640214156296294) + | > grad_norm_0: tensor(27.3691, device='cuda:0') (tensor(21.5874, device='cuda:0')) + | > loss_gen: 2.117471218109131 (2.066981733855554) + | > loss_kl: 2.5374724864959717 (2.512241340434002) + | > loss_feat: 4.568102836608887 (4.455059088459893) + | > loss_mel: 19.985857009887695 (20.456498116038315) + | > loss_duration: 1.348289966583252 (1.3376595890262215) + | > amp_scaler: 64.0 (71.1283292978209) + | > loss_1: 30.557193756103516 (30.828439818744798) + | > grad_norm_1: tensor(1253.8954, device='cuda:0') (tensor(873.6409, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 1.0068 (0.8812811553622563) + | > loader_time: 0.016 (0.013549611297127121) + + + --> TIME: 2024-06-25 12:10:36 -- STEP: 438/591 -- GLOBAL_STEP: 34125 + | > loss_disc: 2.5928032398223877 (2.638422567006116) + | > loss_disc_real_0: 0.20674172043800354 (0.19891295190815508) + | > loss_disc_real_1: 0.2449796497821808 (0.22337795297304788) + | > loss_disc_real_2: 0.2628174126148224 (0.23194645797689212) + | > loss_disc_real_3: 0.26795539259910583 (0.2330248482083077) + | > loss_disc_real_4: 0.2024049311876297 (0.21619803531597195) + | > loss_disc_real_5: 0.23171795904636383 (0.23823792228823928) + | > loss_0: 2.5928032398223877 (2.638422567006116) + | > grad_norm_0: tensor(32.9546, device='cuda:0') (tensor(22.0131, device='cuda:0')) + | > loss_gen: 1.9118963479995728 (2.0692287580063353) + | > loss_kl: 2.43436598777771 (2.509347172632608) + | > loss_feat: 4.220337867736816 (4.454284081720329) + | > loss_mel: 20.079008102416992 (20.452058091011214) + | > loss_duration: 1.3773882389068604 (1.3387297527975133) + | > amp_scaler: 64.0 (70.72146118721467) + | > loss_1: 30.022994995117188 (30.823647799557204) + | > grad_norm_1: tensor(1239.2990, device='cuda:0') (tensor(892.3584, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 1.04 (0.8886823926342132) + | > loader_time: 0.0168 (0.013805423153045514) + + + --> TIME: 2024-06-25 12:11:02 -- STEP: 463/591 -- GLOBAL_STEP: 34150 + | > loss_disc: 2.559154748916626 (2.6370960280910687) + | > loss_disc_real_0: 0.2756882905960083 (0.19869377132561497) + | > loss_disc_real_1: 0.1978335976600647 (0.22321363350227635) + | > loss_disc_real_2: 0.214339017868042 (0.23185376964557763) + | > loss_disc_real_3: 0.22691449522972107 (0.2329770574430668) + | > loss_disc_real_4: 0.22814804315567017 (0.21583998314312158) + | > loss_disc_real_5: 0.25157445669174194 (0.23818324342144745) + | > loss_0: 2.559154748916626 (2.6370960280910687) + | > grad_norm_0: tensor(31.7546, device='cuda:0') (tensor(22.1811, device='cuda:0')) + | > loss_gen: 2.054579973220825 (2.0698010347316886) + | > loss_kl: 2.5886106491088867 (2.5070512789114496) + | > loss_feat: 4.527259349822998 (4.448635481346248) + | > loss_mel: 20.149564743041992 (20.435221398108457) + | > loss_duration: 1.3502354621887207 (1.3398437937701486) + | > amp_scaler: 64.0 (70.35853131749464) + | > loss_1: 30.670249938964844 (30.800552928421975) + | > grad_norm_1: tensor(1271.1222, device='cuda:0') (tensor(903.4907, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 1.0607 (0.896121561141045) + | > loader_time: 0.0182 (0.014104667029164523) + + + --> TIME: 2024-06-25 12:11:29 -- STEP: 488/591 -- GLOBAL_STEP: 34175 + | > loss_disc: 2.6188621520996094 (2.6366484878493144) + | > loss_disc_real_0: 0.24342310428619385 (0.19883864003493157) + | > loss_disc_real_1: 0.21183885633945465 (0.2231844339336528) + | > loss_disc_real_2: 0.23902733623981476 (0.2319088532116081) + | > loss_disc_real_3: 0.23925036191940308 (0.23304622846304396) + | > loss_disc_real_4: 0.18056844174861908 (0.21489992018666912) + | > loss_disc_real_5: 0.2361581027507782 (0.2381285863516272) + | > loss_0: 2.6188621520996094 (2.6366484878493144) + | > grad_norm_0: tensor(22.4495, device='cuda:0') (tensor(22.5361, device='cuda:0')) + | > loss_gen: 2.2853732109069824 (2.0727216545187046) + | > loss_kl: 2.458949089050293 (2.5054559458474635) + | > loss_feat: 4.524751663208008 (4.446696328823686) + | > loss_mel: 20.385915756225586 (20.4292854402886) + | > loss_duration: 1.3950185775756836 (1.3413971953216137) + | > amp_scaler: 64.0 (70.03278688524593) + | > loss_1: 31.050010681152344 (30.79555651398956) + | > grad_norm_1: tensor(1403.6724, device='cuda:0') (tensor(914.0303, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 1.0376 (0.9039761282381464) + | > loader_time: 0.0233 (0.014419840984657163) + + + --> TIME: 2024-06-25 12:11:58 -- STEP: 513/591 -- GLOBAL_STEP: 34200 + | > loss_disc: 2.69380784034729 (2.6376891675283574) + | > loss_disc_real_0: 0.14478519558906555 (0.19891366574615527) + | > loss_disc_real_1: 0.21623536944389343 (0.22317657301649016) + | > loss_disc_real_2: 0.2710415720939636 (0.2319276852798276) + | > loss_disc_real_3: 0.25582581758499146 (0.23321794391724104) + | > loss_disc_real_4: 0.1852613240480423 (0.21526228450601792) + | > loss_disc_real_5: 0.26236650347709656 (0.23825612524796647) + | > loss_0: 2.69380784034729 (2.6376891675283574) + | > grad_norm_0: tensor(32.9047, device='cuda:0') (tensor(22.6097, device='cuda:0')) + | > loss_gen: 2.095625638961792 (2.0707364393721304) + | > loss_kl: 2.4928393363952637 (2.504525788810982) + | > loss_feat: 4.249343395233154 (4.434715476417171) + | > loss_mel: 20.214542388916016 (20.42514325117739) + | > loss_duration: 1.37753427028656 (1.3432988867192697) + | > amp_scaler: 64.0 (69.73879142300198) + | > loss_1: 30.429885864257812 (30.778419814379358) + | > grad_norm_1: tensor(1499.8293, device='cuda:0') (tensor(917.3957, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 1.1593 (0.9145876097167909) + | > loader_time: 0.0183 (0.014710222303751151) + + + --> TIME: 2024-06-25 12:12:28 -- STEP: 538/591 -- GLOBAL_STEP: 34225 + | > loss_disc: 2.5367233753204346 (2.6369188691603656) + | > loss_disc_real_0: 0.13090215623378754 (0.19901879212116214) + | > loss_disc_real_1: 0.20901542901992798 (0.22328017532936262) + | > loss_disc_real_2: 0.2206268608570099 (0.23191487783614587) + | > loss_disc_real_3: 0.2213694453239441 (0.2331637582260437) + | > loss_disc_real_4: 0.21121421456336975 (0.21452387706220816) + | > loss_disc_real_5: 0.23638829588890076 (0.23837639751487505) + | > loss_0: 2.5367233753204346 (2.6369188691603656) + | > grad_norm_0: tensor(28.8122, device='cuda:0') (tensor(22.8627, device='cuda:0')) + | > loss_gen: 2.4922921657562256 (2.0734793324452783) + | > loss_kl: 2.423635482788086 (2.5012536071047013) + | > loss_feat: 4.578187465667725 (4.432346400275108) + | > loss_mel: 20.33204460144043 (20.415409999265982) + | > loss_duration: 1.3876546621322632 (1.3448758278194406) + | > amp_scaler: 64.0 (69.47211895910783) + | > loss_1: 31.213815689086914 (30.767365140099507) + | > grad_norm_1: tensor(1225.2778, device='cuda:0') (tensor(929.6346, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 1.2127 (0.9268951114668723) + | > loader_time: 0.0228 (0.015039544566413285) + + + --> TIME: 2024-06-25 12:12:59 -- STEP: 563/591 -- GLOBAL_STEP: 34250 + | > loss_disc: 2.632551670074463 (2.6383068891867434) + | > loss_disc_real_0: 0.1987827718257904 (0.19921357313015528) + | > loss_disc_real_1: 0.22576755285263062 (0.22364788493191162) + | > loss_disc_real_2: 0.21695950627326965 (0.2319433286812547) + | > loss_disc_real_3: 0.2555878758430481 (0.2332693131683563) + | > loss_disc_real_4: 0.19464614987373352 (0.21408195932582683) + | > loss_disc_real_5: 0.2207578718662262 (0.23845406644191877) + | > loss_0: 2.632551670074463 (2.6383068891867434) + | > grad_norm_0: tensor(24.9386, device='cuda:0') (tensor(23.0687, device='cuda:0')) + | > loss_gen: 2.0054214000701904 (2.072428630258007) + | > loss_kl: 2.48179030418396 (2.498386640955244) + | > loss_feat: 4.3667473793029785 (4.4197168904977) + | > loss_mel: 19.89310073852539 (20.39619662283156) + | > loss_duration: 1.348707914352417 (1.3464699987831588) + | > amp_scaler: 64.0 (69.22912966252224) + | > loss_1: 30.095766067504883 (30.733198758763912) + | > grad_norm_1: tensor(1021.6957, device='cuda:0') (tensor(936.2172, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 1.3127 (0.9402674417512669) + | > loader_time: 0.0254 (0.015429739841982908) + + + --> TIME: 2024-06-25 12:13:32 -- STEP: 588/591 -- GLOBAL_STEP: 34275 + | > loss_disc: 2.8331480026245117 (2.6387930224541902) + | > loss_disc_real_0: 0.4883139133453369 (0.19945830607241916) + | > loss_disc_real_1: 0.2261839061975479 (0.22372148327884225) + | > loss_disc_real_2: 0.2313336282968521 (0.2321169875106033) + | > loss_disc_real_3: 0.21204040944576263 (0.23336542192466406) + | > loss_disc_real_4: 0.20734275877475739 (0.21381716294606937) + | > loss_disc_real_5: 0.24376410245895386 (0.23849930158075022) + | > loss_0: 2.8331480026245117 (2.6387930224541902) + | > grad_norm_0: tensor(62.0690, device='cuda:0') (tensor(23.3225, device='cuda:0')) + | > loss_gen: 1.7915549278259277 (2.072635067766214) + | > loss_kl: 2.465644359588623 (2.495205502526288) + | > loss_feat: 3.95754075050354 (4.410128200946211) + | > loss_mel: 19.696943283081055 (20.38649078615669) + | > loss_duration: 1.3878965377807617 (1.3484694963815258) + | > amp_scaler: 64.0 (69.00680272108845) + | > loss_1: 29.299579620361328 (30.71292901850071) + | > grad_norm_1: tensor(985.0410, device='cuda:0') (tensor(944.1167, device='cuda:0')) + | > current_lr_0: 0.0001985799760895741 + | > current_lr_1: 0.0001985799760895741 + | > step_time: 1.2471 (0.954070101789877) + | > loader_time: 0.0219 (0.01581526451370343) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003688558619073097 (+0.00013130269152052858) + | > avg_loss_disc: 2.714378564915758 (+0.013891742584553057) + | > avg_loss_disc_real_0: 0.3329509940553218 (+0.1529823424334221) + | > avg_loss_disc_real_1: 0.2257625657193204 (-0.0004295259714126587) + | > avg_loss_disc_real_2: 0.22199402432492438 (+0.01667638471786012) + | > avg_loss_disc_real_3: 0.21186949407800715 (+0.01523410195999958) + | > avg_loss_disc_real_4: 0.15941978832508655 (-0.08747544123771345) + | > avg_loss_disc_real_5: 0.22881027897621722 (-0.015492410101789117) + | > avg_loss_0: 2.714378564915758 (+0.013891742584553057) + | > avg_loss_gen: 2.025295800351082 (+0.2010175395519176) + | > avg_loss_kl: 2.5805252460723236 (+0.03305398656966929) + | > avg_loss_feat: 4.103601628161492 (+0.018783488172166685) + | > avg_loss_mel: 20.175799430684844 (-0.05289682428887943) + | > avg_loss_duration: 1.361082145508299 (-0.0037088520983437068) + | > avg_loss_1: 30.246304451151097 (+0.19624969807076909) + + + > EPOCH: 59/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 12:13:44)  + + --> TIME: 2024-06-25 12:14:03 -- STEP: 22/591 -- GLOBAL_STEP: 34300 + | > loss_disc: 2.6273193359375 (2.6636477708816524) + | > loss_disc_real_0: 0.2153094857931137 (0.19976057518612256) + | > loss_disc_real_1: 0.22276417911052704 (0.22805703634565527) + | > loss_disc_real_2: 0.2119787186384201 (0.2345125830986283) + | > loss_disc_real_3: 0.1843566596508026 (0.23461296206170862) + | > loss_disc_real_4: 0.21389487385749817 (0.21455016122622925) + | > loss_disc_real_5: 0.22655589878559113 (0.23823524063283746) + | > loss_0: 2.6273193359375 (2.6636477708816524) + | > grad_norm_0: tensor(17.6677, device='cuda:0') (tensor(20.8354, device='cuda:0')) + | > loss_gen: 2.003866672515869 (2.086967815052379) + | > loss_kl: 2.613140106201172 (2.597810051657937) + | > loss_feat: 4.70106840133667 (4.564636533910577) + | > loss_mel: 21.009571075439453 (20.779699238863856) + | > loss_duration: 1.3027174472808838 (1.313426218249581) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.63036346435547 (31.34253961389715) + | > grad_norm_1: tensor(1040.8761, device='cuda:0') (tensor(915.7162, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.7874 (0.7736597928133878) + | > loader_time: 0.0093 (0.008818756450306288) + + + --> TIME: 2024-06-25 12:14:23 -- STEP: 47/591 -- GLOBAL_STEP: 34325 + | > loss_disc: 2.52807354927063 (2.630273443587283) + | > loss_disc_real_0: 0.27840691804885864 (0.1958576998812087) + | > loss_disc_real_1: 0.22307083010673523 (0.22261079036174936) + | > loss_disc_real_2: 0.2378477156162262 (0.23339206138823895) + | > loss_disc_real_3: 0.26091593503952026 (0.23405036552155273) + | > loss_disc_real_4: 0.13047218322753906 (0.21278724803569468) + | > loss_disc_real_5: 0.2211417257785797 (0.23711210584386866) + | > loss_0: 2.52807354927063 (2.630273443587283) + | > grad_norm_0: tensor(31.7432, device='cuda:0') (tensor(20.4314, device='cuda:0')) + | > loss_gen: 2.173959970474243 (2.0856211616637848) + | > loss_kl: 2.667375326156616 (2.568269541923037) + | > loss_feat: 5.038774013519287 (4.553094990709994) + | > loss_mel: 20.57746696472168 (20.5994812985684) + | > loss_duration: 1.33255934715271 (1.3216956600229794) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.79013442993164 (31.128162546360745) + | > grad_norm_1: tensor(1624.9712, device='cuda:0') (tensor(967.9326, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.8002 (0.7889300001428482) + | > loader_time: 0.0108 (0.009436394305939375) + + + --> TIME: 2024-06-25 12:14:44 -- STEP: 72/591 -- GLOBAL_STEP: 34350 + | > loss_disc: 2.6463961601257324 (2.624805099434323) + | > loss_disc_real_0: 0.1664203405380249 (0.19405501936044958) + | > loss_disc_real_1: 0.21669431030750275 (0.22250771646698317) + | > loss_disc_real_2: 0.24305464327335358 (0.2320823199633095) + | > loss_disc_real_3: 0.232803076505661 (0.23317605629563332) + | > loss_disc_real_4: 0.22411634027957916 (0.21097416347927517) + | > loss_disc_real_5: 0.22247296571731567 (0.23741798206335968) + | > loss_0: 2.6463961601257324 (2.624805099434323) + | > grad_norm_0: tensor(24.5252, device='cuda:0') (tensor(22.1583, device='cuda:0')) + | > loss_gen: 1.991698980331421 (2.0942317843437195) + | > loss_kl: 2.5338640213012695 (2.553803440597323) + | > loss_feat: 4.434093952178955 (4.5703080594539625) + | > loss_mel: 20.95928192138672 (20.607850101259018) + | > loss_duration: 1.324535608291626 (1.3268751137786445) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.24347496032715 (31.153068436516655) + | > grad_norm_1: tensor(1162.9353, device='cuda:0') (tensor(1015.4121, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.8272 (0.7979328234990438) + | > loader_time: 0.0101 (0.009745978646808205) + + + --> TIME: 2024-06-25 12:15:05 -- STEP: 97/591 -- GLOBAL_STEP: 34375 + | > loss_disc: 2.6264710426330566 (2.6264637352265026) + | > loss_disc_real_0: 0.1615261733531952 (0.195173113309231) + | > loss_disc_real_1: 0.1889800727367401 (0.22186744412810533) + | > loss_disc_real_2: 0.20236685872077942 (0.23163149544258707) + | > loss_disc_real_3: 0.21356314420700073 (0.2326880842447281) + | > loss_disc_real_4: 0.1929183304309845 (0.21362986300409453) + | > loss_disc_real_5: 0.24706995487213135 (0.2375968507270223) + | > loss_0: 2.6264710426330566 (2.6264637352265026) + | > grad_norm_0: tensor(20.2866, device='cuda:0') (tensor(22.3567, device='cuda:0')) + | > loss_gen: 2.199190616607666 (2.0887127192979023) + | > loss_kl: 2.615419387817383 (2.5473877031778556) + | > loss_feat: 4.668571949005127 (4.551797520254072) + | > loss_mel: 20.350566864013672 (20.589970736159497) + | > loss_duration: 1.3368884325027466 (1.3338951400874821) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.170637130737305 (31.111763747697022) + | > grad_norm_1: tensor(1069.8275, device='cuda:0') (tensor(1009.1107, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.8298 (0.8057620205830053) + | > loader_time: 0.0105 (0.01004053145339809) + + + --> TIME: 2024-06-25 12:15:26 -- STEP: 122/591 -- GLOBAL_STEP: 34400 + | > loss_disc: 2.623802423477173 (2.626299451609126) + | > loss_disc_real_0: 0.2726421356201172 (0.19596787407749988) + | > loss_disc_real_1: 0.19427801668643951 (0.2213508403692089) + | > loss_disc_real_2: 0.22711557149887085 (0.23135914824536588) + | > loss_disc_real_3: 0.26344019174575806 (0.23366031453746264) + | > loss_disc_real_4: 0.2407774180173874 (0.21347258423195511) + | > loss_disc_real_5: 0.22452887892723083 (0.2377147036986273) + | > loss_0: 2.623802423477173 (2.626299451609126) + | > grad_norm_0: tensor(30.9104, device='cuda:0') (tensor(22.6541, device='cuda:0')) + | > loss_gen: 2.011993408203125 (2.0867097651372206) + | > loss_kl: 2.436115026473999 (2.5389527356038326) + | > loss_feat: 4.6750311851501465 (4.536695263424854) + | > loss_mel: 20.283369064331055 (20.57148383093663) + | > loss_duration: 1.350682258605957 (1.3348320175389778) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.757190704345703 (31.068673493432218) + | > grad_norm_1: tensor(1401.3956, device='cuda:0') (tensor(1019.3266, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.8637 (0.8127863504847542) + | > loader_time: 0.0123 (0.01033705375233635) + + + --> TIME: 2024-06-25 12:15:48 -- STEP: 147/591 -- GLOBAL_STEP: 34425 + | > loss_disc: 2.6422224044799805 (2.6220055080595466) + | > loss_disc_real_0: 0.19962327182292938 (0.19602523536300984) + | > loss_disc_real_1: 0.1935180276632309 (0.22127447667576017) + | > loss_disc_real_2: 0.21438273787498474 (0.23126722264046573) + | > loss_disc_real_3: 0.21469049155712128 (0.2329945965689056) + | > loss_disc_real_4: 0.27372562885284424 (0.2112804064337088) + | > loss_disc_real_5: 0.2514207661151886 (0.23782813092883753) + | > loss_0: 2.6422224044799805 (2.6220055080595466) + | > grad_norm_0: tensor(26.5821, device='cuda:0') (tensor(23.6182, device='cuda:0')) + | > loss_gen: 2.2718348503112793 (2.0948942275274383) + | > loss_kl: 2.464515209197998 (2.5285301857254128) + | > loss_feat: 4.6721320152282715 (4.552439191714433) + | > loss_mel: 20.586891174316406 (20.554772811682056) + | > loss_duration: 1.2965601682662964 (1.334143954880384) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.291933059692383 (31.064780345579393) + | > grad_norm_1: tensor(1620.2592, device='cuda:0') (tensor(1062.5874, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.844 (0.8203260622867922) + | > loader_time: 0.0114 (0.01065788950238909) + + + --> TIME: 2024-06-25 12:16:10 -- STEP: 172/591 -- GLOBAL_STEP: 34450 + | > loss_disc: 2.617095470428467 (2.6232497068338607) + | > loss_disc_real_0: 0.19500838220119476 (0.1964260145242131) + | > loss_disc_real_1: 0.23851646482944489 (0.22156322972719059) + | > loss_disc_real_2: 0.22168594598770142 (0.2314106407900189) + | > loss_disc_real_3: 0.26580503582954407 (0.23304552021761274) + | > loss_disc_real_4: 0.22850742936134338 (0.21092285726999127) + | > loss_disc_real_5: 0.26639506220817566 (0.23802834523971692) + | > loss_0: 2.617095470428467 (2.6232497068338607) + | > grad_norm_0: tensor(27.6630, device='cuda:0') (tensor(23.6950, device='cuda:0')) + | > loss_gen: 1.9668395519256592 (2.0918380424033782) + | > loss_kl: 2.548152208328247 (2.525506600391033) + | > loss_feat: 4.210943698883057 (4.545761206815406) + | > loss_mel: 20.778905868530273 (20.54681008361108) + | > loss_duration: 1.3265918493270874 (1.3331679722597434) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.831432342529297 (31.043083944986034) + | > grad_norm_1: tensor(747.5834, device='cuda:0') (tensor(1060.3451, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.8829 (0.8262858626454376) + | > loader_time: 0.0136 (0.010887254116147065) + + + --> TIME: 2024-06-25 12:16:32 -- STEP: 197/591 -- GLOBAL_STEP: 34475 + | > loss_disc: 2.616593360900879 (2.621757895813376) + | > loss_disc_real_0: 0.11809982359409332 (0.19634022669592485) + | > loss_disc_real_1: 0.2146880328655243 (0.2214413837430441) + | > loss_disc_real_2: 0.21702751517295837 (0.23126091176483232) + | > loss_disc_real_3: 0.21152684092521667 (0.23309692241213648) + | > loss_disc_real_4: 0.2100004404783249 (0.2104777070471478) + | > loss_disc_real_5: 0.2533973157405853 (0.23782593995181436) + | > loss_0: 2.616593360900879 (2.621757895813376) + | > grad_norm_0: tensor(35.4196, device='cuda:0') (tensor(24.0888, device='cuda:0')) + | > loss_gen: 2.464498519897461 (2.0975814428426314) + | > loss_kl: 2.4609155654907227 (2.5197438201323368) + | > loss_feat: 4.985489845275879 (4.561799578255202) + | > loss_mel: 20.47393798828125 (20.545620284104718) + | > loss_duration: 1.3488571643829346 (1.3323548977750213) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.733699798583984 (31.057100063653163) + | > grad_norm_1: tensor(1861.8856, device='cuda:0') (tensor(1089.7673, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.8818 (0.8315762626338126) + | > loader_time: 0.0119 (0.01108928138229448) + + + --> TIME: 2024-06-25 12:16:55 -- STEP: 222/591 -- GLOBAL_STEP: 34500 + | > loss_disc: 2.6948704719543457 (2.622209421149246) + | > loss_disc_real_0: 0.24529430270195007 (0.19658049699422475) + | > loss_disc_real_1: 0.24304313957691193 (0.22145952499127602) + | > loss_disc_real_2: 0.23136727511882782 (0.23124369582882873) + | > loss_disc_real_3: 0.257529616355896 (0.2330940516816603) + | > loss_disc_real_4: 0.22149023413658142 (0.21042858278966164) + | > loss_disc_real_5: 0.2597641348838806 (0.23780706256359546) + | > loss_0: 2.6948704719543457 (2.622209421149246) + | > grad_norm_0: tensor(18.5893, device='cuda:0') (tensor(24.2477, device='cuda:0')) + | > loss_gen: 2.0300164222717285 (2.0948596279900364) + | > loss_kl: 2.3325870037078857 (2.5137964895179667) + | > loss_feat: 4.339392185211182 (4.559940880483334) + | > loss_mel: 20.70628547668457 (20.54198441204725) + | > loss_duration: 1.3303050994873047 (1.3318781820503447) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.73858642578125 (31.042459616789948) + | > grad_norm_1: tensor(931.2001, device='cuda:0') (tensor(1089.6399, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.8879 (0.8369842123340916) + | > loader_time: 0.0153 (0.011394559800087873) + + + --> TIME: 2024-06-25 12:17:17 -- STEP: 247/591 -- GLOBAL_STEP: 34525 + | > loss_disc: 2.579197406768799 (2.6277670387314402) + | > loss_disc_real_0: 0.20833566784858704 (0.19822407079491056) + | > loss_disc_real_1: 0.24665166437625885 (0.22171516963827465) + | > loss_disc_real_2: 0.26511070132255554 (0.23112932987782636) + | > loss_disc_real_3: 0.25252532958984375 (0.2332371101084991) + | > loss_disc_real_4: 0.22035309672355652 (0.2117818593375596) + | > loss_disc_real_5: 0.2436806708574295 (0.2379145170995581) + | > loss_0: 2.579197406768799 (2.6277670387314402) + | > grad_norm_0: tensor(31.7253, device='cuda:0') (tensor(24.2300, device='cuda:0')) + | > loss_gen: 2.27925705909729 (2.0917164531313936) + | > loss_kl: 2.4820291996002197 (2.5134746130661436) + | > loss_feat: 4.840810298919678 (4.5537291297063165) + | > loss_mel: 20.817821502685547 (20.557505047755694) + | > loss_duration: 1.3513686656951904 (1.332364595370737) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.771286010742188 (31.048789846752335) + | > grad_norm_1: tensor(1203.7410, device='cuda:0') (tensor(1073.8904, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.9186 (0.8422311153489086) + | > loader_time: 0.0165 (0.011716221990855603) + + + --> TIME: 2024-06-25 12:17:40 -- STEP: 272/591 -- GLOBAL_STEP: 34550 + | > loss_disc: 2.628472089767456 (2.626370287993376) + | > loss_disc_real_0: 0.20578977465629578 (0.19818634876762242) + | > loss_disc_real_1: 0.22353991866111755 (0.2216053081347662) + | > loss_disc_real_2: 0.21697430312633514 (0.23091046473778348) + | > loss_disc_real_3: 0.25385361909866333 (0.23317582099972403) + | > loss_disc_real_4: 0.18204136192798615 (0.21104077036109042) + | > loss_disc_real_5: 0.23446083068847656 (0.23815526090124073) + | > loss_0: 2.628472089767456 (2.626370287993376) + | > grad_norm_0: tensor(31.7067, device='cuda:0') (tensor(24.5820, device='cuda:0')) + | > loss_gen: 2.1208486557006836 (2.092734005959595) + | > loss_kl: 2.530454635620117 (2.510655960615942) + | > loss_feat: 4.329814910888672 (4.552346391712917) + | > loss_mel: 20.456689834594727 (20.538559338625745) + | > loss_duration: 1.3450299501419067 (1.3324545424650702) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.782838821411133 (31.026750263045816) + | > grad_norm_1: tensor(1571.8392, device='cuda:0') (tensor(1087.6716, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.923 (0.8480159447473639) + | > loader_time: 0.0154 (0.011987282949335439) + + + --> TIME: 2024-06-25 12:18:04 -- STEP: 297/591 -- GLOBAL_STEP: 34575 + | > loss_disc: 2.6499719619750977 (2.6240444793444295) + | > loss_disc_real_0: 0.21746179461479187 (0.19793981702540459) + | > loss_disc_real_1: 0.2390415370464325 (0.22166665676065567) + | > loss_disc_real_2: 0.2325798124074936 (0.23083128958238094) + | > loss_disc_real_3: 0.22459889948368073 (0.2332109388478276) + | > loss_disc_real_4: 0.2406895011663437 (0.21028455409538072) + | > loss_disc_real_5: 0.23062585294246674 (0.23780672954589832) + | > loss_0: 2.6499719619750977 (2.6240444793444295) + | > grad_norm_0: tensor(28.7360, device='cuda:0') (tensor(24.8370, device='cuda:0')) + | > loss_gen: 2.072197437286377 (2.0966097373352297) + | > loss_kl: 2.520185708999634 (2.5080118540561545) + | > loss_feat: 4.519974708557129 (4.556949191623262) + | > loss_mel: 20.593320846557617 (20.54413007485747) + | > loss_duration: 1.3094820976257324 (1.3331790548382387) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.015159606933594 (31.038879933983388) + | > grad_norm_1: tensor(1170.3020, device='cuda:0') (tensor(1094.7349, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.9747 (0.8536865406164817) + | > loader_time: 0.0165 (0.01227926485466235) + + + --> TIME: 2024-06-25 12:18:28 -- STEP: 322/591 -- GLOBAL_STEP: 34600 + | > loss_disc: 2.5751450061798096 (2.623115769824627) + | > loss_disc_real_0: 0.15398931503295898 (0.19754279055488036) + | > loss_disc_real_1: 0.23140345513820648 (0.22187859703850302) + | > loss_disc_real_2: 0.24849680066108704 (0.2308354405049952) + | > loss_disc_real_3: 0.23783770203590393 (0.2330963342271236) + | > loss_disc_real_4: 0.1785767376422882 (0.20981454460517232) + | > loss_disc_real_5: 0.22784090042114258 (0.23809098822544822) + | > loss_0: 2.5751450061798096 (2.623115769824627) + | > grad_norm_0: tensor(28.2798, device='cuda:0') (tensor(25.0368, device='cuda:0')) + | > loss_gen: 2.3093435764312744 (2.097398692776697) + | > loss_kl: 2.3947319984436035 (2.505771220841022) + | > loss_feat: 4.652876853942871 (4.555584292974531) + | > loss_mel: 20.504484176635742 (20.55907598341475) + | > loss_duration: 1.2847040891647339 (1.3337914540160516) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.146141052246094 (31.05162168585736) + | > grad_norm_1: tensor(1420.0366, device='cuda:0') (tensor(1107.2441, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.9329 (0.8596872298613839) + | > loader_time: 0.0167 (0.012528968153532993) + + + --> TIME: 2024-06-25 12:18:51 -- STEP: 347/591 -- GLOBAL_STEP: 34625 + | > loss_disc: 2.62392520904541 (2.624816222561884) + | > loss_disc_real_0: 0.18825595080852509 (0.19799710318489103) + | > loss_disc_real_1: 0.2383403182029724 (0.22209012229263955) + | > loss_disc_real_2: 0.2347421795129776 (0.23133988583981133) + | > loss_disc_real_3: 0.22099360823631287 (0.23316609679106676) + | > loss_disc_real_4: 0.20023243129253387 (0.20966233970108925) + | > loss_disc_real_5: 0.24343788623809814 (0.238102120092348) + | > loss_0: 2.62392520904541 (2.624816222561884) + | > grad_norm_0: tensor(17.3838, device='cuda:0') (tensor(25.3741, device='cuda:0')) + | > loss_gen: 1.985040545463562 (2.0954529308104726) + | > loss_kl: 2.585045099258423 (2.5016520394371975) + | > loss_feat: 4.4466633796691895 (4.543210314055344) + | > loss_mel: 20.286251068115234 (20.552746726044315) + | > loss_duration: 1.3499138355255127 (1.3352852139074451) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.65291404724121 (31.028347262731547) + | > grad_norm_1: tensor(916.8311, device='cuda:0') (tensor(1117.2124, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.9263 (0.8647824261305311) + | > loader_time: 0.0141 (0.012802413286324543) + + + --> TIME: 2024-06-25 12:19:16 -- STEP: 372/591 -- GLOBAL_STEP: 34650 + | > loss_disc: 2.7316722869873047 (2.624936243539216) + | > loss_disc_real_0: 0.18890702724456787 (0.19799876699764882) + | > loss_disc_real_1: 0.24422287940979004 (0.2221338162899658) + | > loss_disc_real_2: 0.2532036304473877 (0.2314659473636458) + | > loss_disc_real_3: 0.28558290004730225 (0.23341389997832238) + | > loss_disc_real_4: 0.21151745319366455 (0.2089050921301048) + | > loss_disc_real_5: 0.24161188304424286 (0.23804528766902544) + | > loss_0: 2.7316722869873047 (2.624936243539216) + | > grad_norm_0: tensor(46.8565, device='cuda:0') (tensor(25.4840, device='cuda:0')) + | > loss_gen: 1.985567569732666 (2.0957693223671265) + | > loss_kl: 2.4227797985076904 (2.4986022563390833) + | > loss_feat: 4.139588832855225 (4.5393844637819525) + | > loss_mel: 19.664304733276367 (20.539996101010217) + | > loss_duration: 1.3516570329666138 (1.3369506738519157) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.56389808654785 (31.01070285612537) + | > grad_norm_1: tensor(930.0090, device='cuda:0') (tensor(1127.1189, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.9713 (0.8701310151366777) + | > loader_time: 0.0179 (0.013099596064577826) + + + --> TIME: 2024-06-25 12:19:40 -- STEP: 397/591 -- GLOBAL_STEP: 34675 + | > loss_disc: 2.4916980266571045 (2.6232517444216645) + | > loss_disc_real_0: 0.2024364173412323 (0.19801986014812958) + | > loss_disc_real_1: 0.1915404498577118 (0.22215001998229952) + | > loss_disc_real_2: 0.24550345540046692 (0.23143595971298458) + | > loss_disc_real_3: 0.25226643681526184 (0.23325114068516556) + | > loss_disc_real_4: 0.1485312581062317 (0.20847797542059762) + | > loss_disc_real_5: 0.24171897768974304 (0.2380754576071684) + | > loss_0: 2.4916980266571045 (2.6232517444216645) + | > grad_norm_0: tensor(28.6900, device='cuda:0') (tensor(26.1019, device='cuda:0')) + | > loss_gen: 2.26995587348938 (2.1010016052789005) + | > loss_kl: 2.3733882904052734 (2.493720982477106) + | > loss_feat: 4.7968950271606445 (4.542401351015877) + | > loss_mel: 20.31761932373047 (20.529603813997735) + | > loss_duration: 1.3608942031860352 (1.337859844080447) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.118755340576172 (31.00458762507595) + | > grad_norm_1: tensor(1281.8445, device='cuda:0') (tensor(1131.9852, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.9671 (0.8759328928642369) + | > loader_time: 0.0155 (0.013369504991046131) + + + --> TIME: 2024-06-25 12:20:05 -- STEP: 422/591 -- GLOBAL_STEP: 34700 + | > loss_disc: 2.668872833251953 (2.62482168154694) + | > loss_disc_real_0: 0.24040746688842773 (0.1981958890978194) + | > loss_disc_real_1: 0.2123977690935135 (0.22216688092992204) + | > loss_disc_real_2: 0.22482162714004517 (0.2313765146509166) + | > loss_disc_real_3: 0.1875194013118744 (0.23308214898358023) + | > loss_disc_real_4: 0.21950674057006836 (0.20907339409558703) + | > loss_disc_real_5: 0.2260235995054245 (0.2379486220072231) + | > loss_0: 2.668872833251953 (2.62482168154694) + | > grad_norm_0: tensor(13.2240, device='cuda:0') (tensor(25.8813, device='cuda:0')) + | > loss_gen: 1.9312033653259277 (2.0963599232700774) + | > loss_kl: 2.53629207611084 (2.4926881422928724) + | > loss_feat: 3.8883025646209717 (4.523742987081342) + | > loss_mel: 20.621726989746094 (20.523987467255065) + | > loss_duration: 1.3465924263000488 (1.3386868430539896) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.32411766052246 (30.975465417473238) + | > grad_norm_1: tensor(229.7992, device='cuda:0') (tensor(1115.8464, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 0.9757 (0.8817976396795698) + | > loader_time: 0.019 (0.013608436448879156) + + + --> TIME: 2024-06-25 12:20:31 -- STEP: 447/591 -- GLOBAL_STEP: 34725 + | > loss_disc: 2.592459201812744 (2.6262433635575135) + | > loss_disc_real_0: 0.24484606087207794 (0.19823649215618236) + | > loss_disc_real_1: 0.22315575182437897 (0.22239451137031752) + | > loss_disc_real_2: 0.25789281725883484 (0.23153531541360306) + | > loss_disc_real_3: 0.23191964626312256 (0.23335264176463655) + | > loss_disc_real_4: 0.21228165924549103 (0.20926296509285644) + | > loss_disc_real_5: 0.25227904319763184 (0.23812373135847267) + | > loss_0: 2.592459201812744 (2.6262433635575135) + | > grad_norm_0: tensor(39.4420, device='cuda:0') (tensor(25.7179, device='cuda:0')) + | > loss_gen: 2.125584125518799 (2.093020127923697) + | > loss_kl: 2.558854103088379 (2.4897588899471614) + | > loss_feat: 4.346848964691162 (4.508040105469812) + | > loss_mel: 20.127071380615234 (20.514673676800133) + | > loss_duration: 1.360512375831604 (1.3398840526073037) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.518869400024414 (30.9453769018186) + | > grad_norm_1: tensor(1762.3636, device='cuda:0') (tensor(1109.4817, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 1.0304 (0.8889204021001555) + | > loader_time: 0.0171 (0.013903259964330622) + + + --> TIME: 2024-06-25 12:20:57 -- STEP: 472/591 -- GLOBAL_STEP: 34750 + | > loss_disc: 2.6154427528381348 (2.626315312365355) + | > loss_disc_real_0: 0.2656848132610321 (0.1983955051055399) + | > loss_disc_real_1: 0.22634346783161163 (0.22240219533569733) + | > loss_disc_real_2: 0.23952963948249817 (0.23149162388832892) + | > loss_disc_real_3: 0.2524062395095825 (0.23332748483171908) + | > loss_disc_real_4: 0.2505628168582916 (0.2092373071048978) + | > loss_disc_real_5: 0.24378910660743713 (0.23812961963526272) + | > loss_0: 2.6154427528381348 (2.626315312365355) + | > grad_norm_0: tensor(49.6135, device='cuda:0') (tensor(25.9811, device='cuda:0')) + | > loss_gen: 2.0868892669677734 (2.0929672374563713) + | > loss_kl: 2.318211078643799 (2.484897595846047) + | > loss_feat: 4.282541275024414 (4.49851299595025) + | > loss_mel: 19.571802139282227 (20.49186763116868) + | > loss_duration: 1.371642827987671 (1.3412473648786545) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.631086349487305 (30.909492864447124) + | > grad_norm_1: tensor(1224.1449, device='cuda:0') (tensor(1108.2119, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 1.0119 (0.8959770808785649) + | > loader_time: 0.0173 (0.014184794183504787) + + + --> TIME: 2024-06-25 12:21:25 -- STEP: 497/591 -- GLOBAL_STEP: 34775 + | > loss_disc: 2.7287726402282715 (2.6261939700220682) + | > loss_disc_real_0: 0.23253598809242249 (0.19840204958584706) + | > loss_disc_real_1: 0.19569329917430878 (0.22250222143392928) + | > loss_disc_real_2: 0.23081155121326447 (0.23149176640529748) + | > loss_disc_real_3: 0.3121311068534851 (0.23321008028638673) + | > loss_disc_real_4: 0.23600749671459198 (0.20916304182658502) + | > loss_disc_real_5: 0.24085158109664917 (0.23806844019074316) + | > loss_0: 2.7287726402282715 (2.6261939700220682) + | > grad_norm_0: tensor(19.9453, device='cuda:0') (tensor(26.0794, device='cuda:0')) + | > loss_gen: 1.8849501609802246 (2.093456689500714) + | > loss_kl: 2.5427494049072266 (2.4831394165814284) + | > loss_feat: 3.954681158065796 (4.494554453452592) + | > loss_mel: 19.152103424072266 (20.478983428156823) + | > loss_duration: 1.3788782358169556 (1.342566757374846) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 28.913362503051758 (30.892700790159658) + | > grad_norm_1: tensor(895.6738, device='cuda:0') (tensor(1108.5403, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 1.1763 (0.9062649109953606) + | > loader_time: 0.023 (0.014508596128621095) + + + --> TIME: 2024-06-25 12:21:54 -- STEP: 522/591 -- GLOBAL_STEP: 34800 + | > loss_disc: 2.5505762100219727 (2.626543855758463) + | > loss_disc_real_0: 0.20896156132221222 (0.1984667038951797) + | > loss_disc_real_1: 0.24271218478679657 (0.2225867294602924) + | > loss_disc_real_2: 0.21768921613693237 (0.23150669460780776) + | > loss_disc_real_3: 0.24323704838752747 (0.23324058153505983) + | > loss_disc_real_4: 0.21068993210792542 (0.209066299600277) + | > loss_disc_real_5: 0.24369940161705017 (0.2379502453680696) + | > loss_0: 2.5505762100219727 (2.626543855758463) + | > grad_norm_0: tensor(36.2936, device='cuda:0') (tensor(26.0872, device='cuda:0')) + | > loss_gen: 2.2897424697875977 (2.0920755070744814) + | > loss_kl: 2.3945178985595703 (2.48134153601767) + | > loss_feat: 4.5723347663879395 (4.487277733411829) + | > loss_mel: 19.875032424926758 (20.469001653093937) + | > loss_duration: 1.3831937313079834 (1.3443575072105818) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.514820098876953 (30.874053984309523) + | > grad_norm_1: tensor(1546.4058, device='cuda:0') (tensor(1105.3434, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 1.1542 (0.9171563848225093) + | > loader_time: 0.0191 (0.014815392622089025) + + + --> TIME: 2024-06-25 12:22:25 -- STEP: 547/591 -- GLOBAL_STEP: 34825 + | > loss_disc: 2.710818290710449 (2.626453979795551) + | > loss_disc_real_0: 0.20323467254638672 (0.1983748613916324) + | > loss_disc_real_1: 0.24382521212100983 (0.22281795279744337) + | > loss_disc_real_2: 0.20946812629699707 (0.23156823727286932) + | > loss_disc_real_3: 0.27078378200531006 (0.23327580821688473) + | > loss_disc_real_4: 0.2120915949344635 (0.20872601631082183) + | > loss_disc_real_5: 0.25261619687080383 (0.23796994485628453) + | > loss_0: 2.710818290710449 (2.626453979795551) + | > grad_norm_0: tensor(13.8467, device='cuda:0') (tensor(26.4312, device='cuda:0')) + | > loss_gen: 1.9674901962280273 (2.094001961364608) + | > loss_kl: 2.5129921436309814 (2.4809354667907657) + | > loss_feat: 4.061674118041992 (4.48467085653511) + | > loss_mel: 19.801494598388672 (20.461305691511576) + | > loss_duration: 1.4048242568969727 (1.3461356108541676) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.74847412109375 (30.86704962693795) + | > grad_norm_1: tensor(147.6650, device='cuda:0') (tensor(1102.7087, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 1.187 (0.9297548390829585) + | > loader_time: 0.0256 (0.015149268197403096) + + + --> TIME: 2024-06-25 12:22:57 -- STEP: 572/591 -- GLOBAL_STEP: 34850 + | > loss_disc: 2.644805431365967 (2.6279268669081746) + | > loss_disc_real_0: 0.1980305314064026 (0.1984412642000439) + | > loss_disc_real_1: 0.17637549340724945 (0.22280539595684803) + | > loss_disc_real_2: 0.21299804747104645 (0.2316528590513276) + | > loss_disc_real_3: 0.20865032076835632 (0.23319512532724368) + | > loss_disc_real_4: 0.22581079602241516 (0.20943891857522778) + | > loss_disc_real_5: 0.22416019439697266 (0.23794980935908697) + | > loss_0: 2.644805431365967 (2.6279268669081746) + | > grad_norm_0: tensor(21.0687, device='cuda:0') (tensor(26.1322, device='cuda:0')) + | > loss_gen: 2.079011917114258 (2.0901522886502995) + | > loss_kl: 2.3874521255493164 (2.4796569334877114) + | > loss_feat: 4.098270893096924 (4.468845130680325) + | > loss_mel: 20.14573860168457 (20.447560880567636) + | > loss_duration: 1.3834904432296753 (1.3477355575228065) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.093963623046875 (30.83395084301075) + | > grad_norm_1: tensor(1101.0015, device='cuda:0') (tensor(1091.4453, device='cuda:0')) + | > current_lr_0: 0.0001985551535925629 + | > current_lr_1: 0.0001985551535925629 + | > step_time: 1.3004 (0.9436766271824604) + | > loader_time: 0.0206 (0.01552149049051992) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035586154207270194 (-0.0001299431983460774) + | > avg_loss_disc: 2.658841640391249 (-0.05553692452450898) + | > avg_loss_disc_real_0: 0.16972551732621294 (-0.16322547672910886) + | > avg_loss_disc_real_1: 0.2058506652395776 (-0.01991190047974281) + | > avg_loss_disc_real_2: 0.22600478124111256 (+0.004010756916188185) + | > avg_loss_disc_real_3: 0.23215175055442971 (+0.020282256476422567) + | > avg_loss_disc_real_4: 0.18101875658365005 (+0.0215989682585635) + | > avg_loss_disc_real_5: 0.23528021827657172 (+0.006469939300354499) + | > avg_loss_0: 2.658841640391249 (-0.05553692452450898) + | > avg_loss_gen: 1.81123138742244 (-0.2140644129286422) + | > avg_loss_kl: 2.546479275886049 (-0.034045970186274754) + | > avg_loss_feat: 4.226584150436077 (+0.12298252227458484) + | > avg_loss_mel: 20.2903444006088 (+0.11454496992395491) + | > avg_loss_duration: 1.3579795208383114 (-0.0031026246699876747) + | > avg_loss_1: 30.232618859473696 (-0.013685591677401021) + + + > EPOCH: 60/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 12:23:29)  + + --> TIME: 2024-06-25 12:23:35 -- STEP: 6/591 -- GLOBAL_STEP: 34875 + | > loss_disc: 2.682568073272705 (2.6241496801376343) + | > loss_disc_real_0: 0.21856865286827087 (0.19464361170927683) + | > loss_disc_real_1: 0.21886955201625824 (0.23299873620271683) + | > loss_disc_real_2: 0.27132827043533325 (0.23829780519008636) + | > loss_disc_real_3: 0.2330579161643982 (0.23960320154825845) + | > loss_disc_real_4: 0.1869853287935257 (0.20119495689868927) + | > loss_disc_real_5: 0.25411900877952576 (0.2465224266052246) + | > loss_0: 2.682568073272705 (2.6241496801376343) + | > grad_norm_0: tensor(30.0746, device='cuda:0') (tensor(29.2938, device='cuda:0')) + | > loss_gen: 1.9506162405014038 (2.154833416144053) + | > loss_kl: 2.604321002960205 (2.599018931388855) + | > loss_feat: 4.713164329528809 (4.902546326319377) + | > loss_mel: 21.225553512573242 (20.989118893941242) + | > loss_duration: 1.3174736499786377 (1.3320831855138142) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.81113052368164 (31.977601369222004) + | > grad_norm_1: tensor(1303.7911, device='cuda:0') (tensor(1311.5422, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.7626 (0.7638015747070312) + | > loader_time: 0.0075 (0.008277654647827148) + + + --> TIME: 2024-06-25 12:23:55 -- STEP: 31/591 -- GLOBAL_STEP: 34900 + | > loss_disc: 2.5936050415039062 (2.6081123428960002) + | > loss_disc_real_0: 0.2960974872112274 (0.19965385669662108) + | > loss_disc_real_1: 0.2556471526622772 (0.22351444776981108) + | > loss_disc_real_2: 0.2244153767824173 (0.23296603416242906) + | > loss_disc_real_3: 0.23281759023666382 (0.23432786906919173) + | > loss_disc_real_4: 0.18842624127864838 (0.2004485067821318) + | > loss_disc_real_5: 0.23086686432361603 (0.23755125749495723) + | > loss_0: 2.5936050415039062 (2.6081123428960002) + | > grad_norm_0: tensor(35.7950, device='cuda:0') (tensor(27.4983, device='cuda:0')) + | > loss_gen: 2.155987501144409 (2.1513057062702794) + | > loss_kl: 2.6079437732696533 (2.6054318566476145) + | > loss_feat: 4.9263763427734375 (4.7690587966672835) + | > loss_mel: 20.9415283203125 (20.64821187911495) + | > loss_duration: 1.366544246673584 (1.3129029504714473) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.998380661010742 (31.48691152757214) + | > grad_norm_1: tensor(1949.4780, device='cuda:0') (tensor(1262.6389, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 1.0167 (0.7850456314702188) + | > loader_time: 0.0095 (0.009102359894783266) + + + --> TIME: 2024-06-25 12:24:16 -- STEP: 56/591 -- GLOBAL_STEP: 34925 + | > loss_disc: 2.6623902320861816 (2.6118399756295334) + | > loss_disc_real_0: 0.16044694185256958 (0.19838793549154485) + | > loss_disc_real_1: 0.2425830066204071 (0.2232325625206743) + | > loss_disc_real_2: 0.20498326420783997 (0.23248581987406527) + | > loss_disc_real_3: 0.20859822630882263 (0.23380212086652005) + | > loss_disc_real_4: 0.20652402937412262 (0.19881372872207845) + | > loss_disc_real_5: 0.22954332828521729 (0.23710332731051104) + | > loss_0: 2.6623902320861816 (2.6118399756295334) + | > grad_norm_0: tensor(33.3011, device='cuda:0') (tensor(28.4402, device='cuda:0')) + | > loss_gen: 2.304666757583618 (2.154586166143416) + | > loss_kl: 2.528916120529175 (2.5650673466069356) + | > loss_feat: 4.776655673980713 (4.726053110190802) + | > loss_mel: 21.256288528442383 (20.603511810302734) + | > loss_duration: 1.3744102716445923 (1.3224345615931916) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.240936279296875 (31.371653148106166) + | > grad_norm_1: tensor(987.1488, device='cuda:0') (tensor(1260.8650, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.8058 (0.7945155160767692) + | > loader_time: 0.0115 (0.00958672591618129) + + + --> TIME: 2024-06-25 12:24:37 -- STEP: 81/591 -- GLOBAL_STEP: 34950 + | > loss_disc: 2.6121678352355957 (2.618460572796102) + | > loss_disc_real_0: 0.24019074440002441 (0.1997378291907134) + | > loss_disc_real_1: 0.22137410938739777 (0.2242164052562949) + | > loss_disc_real_2: 0.24291017651557922 (0.23178475672080193) + | > loss_disc_real_3: 0.23353886604309082 (0.23299187624160153) + | > loss_disc_real_4: 0.18555773794651031 (0.20213364211865414) + | > loss_disc_real_5: 0.2306642234325409 (0.23779963214456298) + | > loss_0: 2.6121678352355957 (2.618460572796102) + | > grad_norm_0: tensor(31.2012, device='cuda:0') (tensor(29.6921, device='cuda:0')) + | > loss_gen: 2.051729679107666 (2.134998818974435) + | > loss_kl: 2.4506711959838867 (2.539667785903554) + | > loss_feat: 4.633547782897949 (4.658726815824155) + | > loss_mel: 20.40448760986328 (20.61954616028586) + | > loss_duration: 1.350368857383728 (1.3282142848144343) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.890804290771484 (31.281153902595427) + | > grad_norm_1: tensor(1602.1849, device='cuda:0') (tensor(1264.8480, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.8377 (0.8028229342566597) + | > loader_time: 0.0123 (0.009981149508629314) + + + --> TIME: 2024-06-25 12:24:58 -- STEP: 106/591 -- GLOBAL_STEP: 34975 + | > loss_disc: 2.6971306800842285 (2.6269122542075385) + | > loss_disc_real_0: 0.19989240169525146 (0.1991855577337292) + | > loss_disc_real_1: 0.2326555699110031 (0.2236947812280565) + | > loss_disc_real_2: 0.2263013869524002 (0.23240735466187856) + | > loss_disc_real_3: 0.28769493103027344 (0.23360099165507084) + | > loss_disc_real_4: 0.20876391232013702 (0.20674262013075487) + | > loss_disc_real_5: 0.26936018466949463 (0.2389516217528649) + | > loss_0: 2.6971306800842285 (2.6269122542075385) + | > grad_norm_0: tensor(14.0777, device='cuda:0') (tensor(28.4304, device='cuda:0')) + | > loss_gen: 1.86323082447052 (2.1160465737558747) + | > loss_kl: 2.4588773250579834 (2.5303323246398053) + | > loss_feat: 3.862291097640991 (4.601608678979689) + | > loss_mel: 19.722440719604492 (20.62044244442346) + | > loss_duration: 1.3551337718963623 (1.3336511726649305) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.261974334716797 (31.202081230451476) + | > grad_norm_1: tensor(787.7424, device='cuda:0') (tensor(1215.2725, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.8385 (0.8096706912202657) + | > loader_time: 0.0119 (0.010292455835162466) + + + --> TIME: 2024-06-25 12:25:19 -- STEP: 131/591 -- GLOBAL_STEP: 35000 + | > loss_disc: 2.657036781311035 (2.620788792617448) + | > loss_disc_real_0: 0.14584292471408844 (0.1977773328091352) + | > loss_disc_real_1: 0.20398937165737152 (0.22305661075897798) + | > loss_disc_real_2: 0.23071102797985077 (0.23170506248947317) + | > loss_disc_real_3: 0.24359607696533203 (0.23305066328012305) + | > loss_disc_real_4: 0.21911726891994476 (0.20580056070826436) + | > loss_disc_real_5: 0.22359302639961243 (0.23805260624139363) + | > loss_0: 2.657036781311035 (2.620788792617448) + | > grad_norm_0: tensor(19.9491, device='cuda:0') (tensor(28.4806, device='cuda:0')) + | > loss_gen: 2.1569976806640625 (2.1168058537344887) + | > loss_kl: 2.4680261611938477 (2.524072577935138) + | > loss_feat: 4.307432174682617 (4.609591103692088) + | > loss_mel: 20.4991512298584 (20.624494144934733) + | > loss_duration: 1.375988483428955 (1.3353095054626467) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.807594299316406 (31.21027320395899) + | > grad_norm_1: tensor(775.1748, device='cuda:0') (tensor(1209.0166, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.8654 (0.8161115682762089) + | > loader_time: 0.0123 (0.01057414608147308) + + + > CHECKPOINT : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/checkpoint_35000.pth + + --> TIME: 2024-06-25 12:25:44 -- STEP: 156/591 -- GLOBAL_STEP: 35025 + | > loss_disc: 2.5386550426483154 (2.623424965601701) + | > loss_disc_real_0: 0.20728129148483276 (0.19800276863269317) + | > loss_disc_real_1: 0.27965524792671204 (0.22295828650777155) + | > loss_disc_real_2: 0.223556786775589 (0.23169748609264693) + | > loss_disc_real_3: 0.24228273332118988 (0.23341375541610596) + | > loss_disc_real_4: 0.1893727332353592 (0.20810800647506342) + | > loss_disc_real_5: 0.22079992294311523 (0.23817525355097574) + | > loss_0: 2.5386550426483154 (2.623424965601701) + | > grad_norm_0: tensor(32.7438, device='cuda:0') (tensor(27.3264, device='cuda:0')) + | > loss_gen: 2.2659716606140137 (2.1067245808931494) + | > loss_kl: 2.5812270641326904 (2.5140997675748964) + | > loss_feat: 4.828464031219482 (4.581267285041316) + | > loss_mel: 20.800537109375 (20.600020812107967) + | > loss_duration: 1.3282098770141602 (1.3338371790372407) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.80440902709961 (31.135949599437225) + | > grad_norm_1: tensor(1322.0358, device='cuda:0') (tensor(1183.5742, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.8881 (0.8224975482011453) + | > loader_time: 0.0113 (0.01081944276125003) + + + --> TIME: 2024-06-25 12:26:06 -- STEP: 181/591 -- GLOBAL_STEP: 35050 + | > loss_disc: 2.6600656509399414 (2.623197347419697) + | > loss_disc_real_0: 0.29649221897125244 (0.1988190498141294) + | > loss_disc_real_1: 0.1961853951215744 (0.22246918814946276) + | > loss_disc_real_2: 0.23956435918807983 (0.23167510503563432) + | > loss_disc_real_3: 0.2528568208217621 (0.23362446531077116) + | > loss_disc_real_4: 0.19360315799713135 (0.2072633517216582) + | > loss_disc_real_5: 0.2517912685871124 (0.23849111968311815) + | > loss_0: 2.6600656509399414 (2.623197347419697) + | > grad_norm_0: tensor(24.3629, device='cuda:0') (tensor(27.4245, device='cuda:0')) + | > loss_gen: 2.0192477703094482 (2.1071988653741482) + | > loss_kl: 2.533182144165039 (2.5124932231165427) + | > loss_feat: 4.443085193634033 (4.585985216646559) + | > loss_mel: 20.47587013244629 (20.57423283908907) + | > loss_duration: 1.319758653640747 (1.3320581623203855) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.7911434173584 (31.111968256491984) + | > grad_norm_1: tensor(1360.3076, device='cuda:0') (tensor(1202.8942, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.8629 (0.8285088802569479) + | > loader_time: 0.0119 (0.010991279591512942) + + + --> TIME: 2024-06-25 12:26:29 -- STEP: 206/591 -- GLOBAL_STEP: 35075 + | > loss_disc: 2.657620429992676 (2.622939494049665) + | > loss_disc_real_0: 0.15403082966804504 (0.19862182633680048) + | > loss_disc_real_1: 0.23345613479614258 (0.22264590410931598) + | > loss_disc_real_2: 0.23163093626499176 (0.23156953610262823) + | > loss_disc_real_3: 0.2317613959312439 (0.2333685231295604) + | > loss_disc_real_4: 0.20580925047397614 (0.20692474452905282) + | > loss_disc_real_5: 0.21585530042648315 (0.23802302683730728) + | > loss_0: 2.657620429992676 (2.622939494049665) + | > grad_norm_0: tensor(38.9926, device='cuda:0') (tensor(27.6033, device='cuda:0')) + | > loss_gen: 2.081127643585205 (2.106688572365104) + | > loss_kl: 2.5617711544036865 (2.5089716020139683) + | > loss_feat: 4.61285924911499 (4.586583694208012) + | > loss_mel: 20.99595069885254 (20.56374435980343) + | > loss_duration: 1.3244562149047852 (1.3313598598091358) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.57616424560547 (31.097348065052216) + | > grad_norm_1: tensor(1548.5251, device='cuda:0') (tensor(1224.7208, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.8916 (0.8340221562431854) + | > loader_time: 0.0137 (0.011184899552354535) + + + --> TIME: 2024-06-25 12:26:51 -- STEP: 231/591 -- GLOBAL_STEP: 35100 + | > loss_disc: 2.5659127235412598 (2.6227275220862714) + | > loss_disc_real_0: 0.14810660481452942 (0.19893365998417784) + | > loss_disc_real_1: 0.22161035239696503 (0.22235195687064876) + | > loss_disc_real_2: 0.23128542304039001 (0.2316499518883693) + | > loss_disc_real_3: 0.24837785959243774 (0.23314613574769075) + | > loss_disc_real_4: 0.23264698684215546 (0.2071771785552367) + | > loss_disc_real_5: 0.2602617144584656 (0.23812713399851992) + | > loss_0: 2.5659127235412598 (2.6227275220862714) + | > grad_norm_0: tensor(37.4899, device='cuda:0') (tensor(28.0755, device='cuda:0')) + | > loss_gen: 2.1937923431396484 (2.1073189705481274) + | > loss_kl: 2.544888496398926 (2.5047808828808016) + | > loss_feat: 4.564117908477783 (4.584633576405511) + | > loss_mel: 19.944995880126953 (20.542752889327673) + | > loss_duration: 1.3374873399734497 (1.3312268484206438) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.585283279418945 (31.0707131670667) + | > grad_norm_1: tensor(1669.7546, device='cuda:0') (tensor(1235.6808, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.8838 (0.8395494227801567) + | > loader_time: 0.0145 (0.011473451341901506) + + + --> TIME: 2024-06-25 12:27:14 -- STEP: 256/591 -- GLOBAL_STEP: 35125 + | > loss_disc: 2.6244611740112305 (2.6240559834986925) + | > loss_disc_real_0: 0.18749628961086273 (0.19906922080554068) + | > loss_disc_real_1: 0.25021588802337646 (0.22239370952593163) + | > loss_disc_real_2: 0.2532598078250885 (0.2315488567110151) + | > loss_disc_real_3: 0.22100363671779633 (0.23305535985855386) + | > loss_disc_real_4: 0.19719715416431427 (0.20775534486165267) + | > loss_disc_real_5: 0.23146049678325653 (0.2384331683279015) + | > loss_0: 2.6244611740112305 (2.6240559834986925) + | > grad_norm_0: tensor(33.1111, device='cuda:0') (tensor(27.8964, device='cuda:0')) + | > loss_gen: 2.008545160293579 (2.105271634645762) + | > loss_kl: 2.5118343830108643 (2.505687502212823) + | > loss_feat: 4.536050796508789 (4.576136865653096) + | > loss_mel: 20.798419952392578 (20.543043933808804) + | > loss_duration: 1.3550934791564941 (1.3312773727811882) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.209943771362305 (31.06141733378172) + | > grad_norm_1: tensor(1239.8512, device='cuda:0') (tensor(1223.5956, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.8924 (0.8452809704467654) + | > loader_time: 0.0151 (0.011735472828149796) + + + --> TIME: 2024-06-25 12:27:37 -- STEP: 281/591 -- GLOBAL_STEP: 35150 + | > loss_disc: 2.668621063232422 (2.6248776149070987) + | > loss_disc_real_0: 0.1410866677761078 (0.1991155070660377) + | > loss_disc_real_1: 0.23644377291202545 (0.22253212958468238) + | > loss_disc_real_2: 0.21720455586910248 (0.2315629048279596) + | > loss_disc_real_3: 0.2003176510334015 (0.23321612879262701) + | > loss_disc_real_4: 0.1750282198190689 (0.20754576402838965) + | > loss_disc_real_5: 0.226144477725029 (0.23825106878509725) + | > loss_0: 2.668621063232422 (2.6248776149070987) + | > grad_norm_0: tensor(36.3224, device='cuda:0') (tensor(28.0230, device='cuda:0')) + | > loss_gen: 2.189763069152832 (2.1048001258822957) + | > loss_kl: 2.412661552429199 (2.5029050073589727) + | > loss_feat: 4.610091686248779 (4.573050885013837) + | > loss_mel: 20.687559127807617 (20.534611237006683) + | > loss_duration: 1.3663463592529297 (1.332002770858304) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.266420364379883 (31.047370041816684) + | > grad_norm_1: tensor(1672.9554, device='cuda:0') (tensor(1229.6758, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.9044 (0.8508371916529969) + | > loader_time: 0.014 (0.012051576397172921) + + + --> TIME: 2024-06-25 12:28:01 -- STEP: 306/591 -- GLOBAL_STEP: 35175 + | > loss_disc: 2.482726812362671 (2.6232667428995264) + | > loss_disc_real_0: 0.1679755449295044 (0.1989250369613467) + | > loss_disc_real_1: 0.25894758105278015 (0.2224724582986894) + | > loss_disc_real_2: 0.25564831495285034 (0.2315907306433503) + | > loss_disc_real_3: 0.25818559527397156 (0.23333393723941317) + | > loss_disc_real_4: 0.1469961553812027 (0.2071855195791892) + | > loss_disc_real_5: 0.1959046721458435 (0.23823723123938428) + | > loss_0: 2.482726812362671 (2.6232667428995264) + | > grad_norm_0: tensor(27.2999, device='cuda:0') (tensor(28.0530, device='cuda:0')) + | > loss_gen: 2.124109983444214 (2.1053098222009505) + | > loss_kl: 2.3598361015319824 (2.5022856799605626) + | > loss_feat: 4.799398899078369 (4.573247900975294) + | > loss_mel: 20.579273223876953 (20.53103648915011) + | > loss_duration: 1.3741393089294434 (1.33240847221387) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.236757278442383 (31.04428835476146) + | > grad_norm_1: tensor(1751.8082, device='cuda:0') (tensor(1236.9186, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.9095 (0.8567070228601589) + | > loader_time: 0.0162 (0.012340408524656605) + + + --> TIME: 2024-06-25 12:28:25 -- STEP: 331/591 -- GLOBAL_STEP: 35200 + | > loss_disc: 2.612351417541504 (2.622297263217474) + | > loss_disc_real_0: 0.23121890425682068 (0.19863500502354428) + | > loss_disc_real_1: 0.19715137779712677 (0.22244443944753836) + | > loss_disc_real_2: 0.21043100953102112 (0.23154877351309958) + | > loss_disc_real_3: 0.24223481118679047 (0.2332850937756887) + | > loss_disc_real_4: 0.24598443508148193 (0.20707850536370917) + | > loss_disc_real_5: 0.23621614277362823 (0.23822247113165898) + | > loss_0: 2.612351417541504 (2.622297263217474) + | > grad_norm_0: tensor(27.5183, device='cuda:0') (tensor(28.2139, device='cuda:0')) + | > loss_gen: 2.034480571746826 (2.1075015521841833) + | > loss_kl: 2.5447049140930176 (2.50080727450437) + | > loss_feat: 4.441136360168457 (4.5783623676645755) + | > loss_mel: 20.28375816345215 (20.538174349735876) + | > loss_duration: 1.3248751163482666 (1.3332698337981344) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.628955841064453 (31.058115385810414) + | > grad_norm_1: tensor(818.4031, device='cuda:0') (tensor(1229.1417, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 1.0037 (0.8624967801246755) + | > loader_time: 0.0154 (0.012614069388352127) + + + --> TIME: 2024-06-25 12:28:49 -- STEP: 356/591 -- GLOBAL_STEP: 35225 + | > loss_disc: 2.710465431213379 (2.6242805766255644) + | > loss_disc_real_0: 0.26798462867736816 (0.19889921393622173) + | > loss_disc_real_1: 0.21036642789840698 (0.22247233103667752) + | > loss_disc_real_2: 0.22536379098892212 (0.23169784783647301) + | > loss_disc_real_3: 0.22066320478916168 (0.23339800628718366) + | > loss_disc_real_4: 0.2423148900270462 (0.207553503553519) + | > loss_disc_real_5: 0.27569299936294556 (0.23823227704073605) + | > loss_0: 2.710465431213379 (2.6242805766255644) + | > grad_norm_0: tensor(26.2935, device='cuda:0') (tensor(28.2227, device='cuda:0')) + | > loss_gen: 1.9154900312423706 (2.104321111454052) + | > loss_kl: 2.474264621734619 (2.499081253335716) + | > loss_feat: 4.10854434967041 (4.566761358400407) + | > loss_mel: 20.083019256591797 (20.529519868700692) + | > loss_duration: 1.3481698036193848 (1.3344542806068151) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.929487228393555 (31.034137897277148) + | > grad_norm_1: tensor(731.2650, device='cuda:0') (tensor(1224.5431, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.9364 (0.8678101321284687) + | > loader_time: 0.0177 (0.012896657659766374) + + + --> TIME: 2024-06-25 12:29:13 -- STEP: 381/591 -- GLOBAL_STEP: 35250 + | > loss_disc: 2.600775718688965 (2.623490262845055) + | > loss_disc_real_0: 0.18341034650802612 (0.19825482681354512) + | > loss_disc_real_1: 0.2603827118873596 (0.22240575244576913) + | > loss_disc_real_2: 0.25285810232162476 (0.2316307642172015) + | > loss_disc_real_3: 0.244798943400383 (0.2332158105110559) + | > loss_disc_real_4: 0.19555434584617615 (0.20833074405124485) + | > loss_disc_real_5: 0.2312702238559723 (0.23810240606779814) + | > loss_0: 2.600775718688965 (2.623490262845055) + | > grad_norm_0: tensor(24.6813, device='cuda:0') (tensor(27.8117, device='cuda:0')) + | > loss_gen: 2.29492449760437 (2.102678900002808) + | > loss_kl: 2.485994815826416 (2.4961044319032677) + | > loss_feat: 4.789735317230225 (4.555535883415401) + | > loss_mel: 20.327646255493164 (20.522659341807127) + | > loss_duration: 1.3667913675308228 (1.3359797861319505) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.265090942382812 (31.012958351395575) + | > grad_norm_1: tensor(1493.2860, device='cuda:0') (tensor(1214.2727, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.964 (0.873711793754357) + | > loader_time: 0.0181 (0.013164439539271072) + + + --> TIME: 2024-06-25 12:29:38 -- STEP: 406/591 -- GLOBAL_STEP: 35275 + | > loss_disc: 2.731278896331787 (2.6239217549122005) + | > loss_disc_real_0: 0.18446822464466095 (0.19813760103923933) + | > loss_disc_real_1: 0.18651536107063293 (0.2224559764483292) + | > loss_disc_real_2: 0.22249756753444672 (0.23174740810846461) + | > loss_disc_real_3: 0.2256949245929718 (0.23334597998064727) + | > loss_disc_real_4: 0.240324005484581 (0.20823763854104302) + | > loss_disc_real_5: 0.23784708976745605 (0.23809311087495588) + | > loss_0: 2.731278896331787 (2.6239217549122005) + | > grad_norm_0: tensor(7.4017, device='cuda:0') (tensor(27.9678, device='cuda:0')) + | > loss_gen: 1.9685134887695312 (2.1017773022205364) + | > loss_kl: 2.450235366821289 (2.4957543775953086) + | > loss_feat: 4.203873157501221 (4.548930580392846) + | > loss_mel: 19.995607376098633 (20.51640325931493) + | > loss_duration: 1.343985915184021 (1.3371350935527246) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.962215423583984 (31.000000629519008) + | > grad_norm_1: tensor(206.5898, device='cuda:0') (tensor(1216.6484, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.9891 (0.879729053069805) + | > loader_time: 0.0195 (0.013449483904345277) + + + --> TIME: 2024-06-25 12:30:03 -- STEP: 431/591 -- GLOBAL_STEP: 35300 + | > loss_disc: 2.6480488777160645 (2.6250155746521897) + | > loss_disc_real_0: 0.11881039291620255 (0.19835436354518768) + | > loss_disc_real_1: 0.19931109249591827 (0.2226221154847997) + | > loss_disc_real_2: 0.21662504971027374 (0.2315870399334038) + | > loss_disc_real_3: 0.23287762701511383 (0.2332840973636667) + | > loss_disc_real_4: 0.18206700682640076 (0.20886832482576917) + | > loss_disc_real_5: 0.25051793456077576 (0.23806562075897048) + | > loss_0: 2.6480488777160645 (2.6250155746521897) + | > grad_norm_0: tensor(26.2393, device='cuda:0') (tensor(27.8683, device='cuda:0')) + | > loss_gen: 2.110396385192871 (2.0985785006370454) + | > loss_kl: 2.4601945877075195 (2.4958034778692317) + | > loss_feat: 4.244076251983643 (4.535263577636045) + | > loss_mel: 20.2838191986084 (20.503229805045784) + | > loss_duration: 1.357529878616333 (1.3380970327040977) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.456018447875977 (30.970972428465664) + | > grad_norm_1: tensor(1441.1943, device='cuda:0') (tensor(1206.4685, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.9814 (0.8860672488290028) + | > loader_time: 0.0181 (0.013731101272830826) + + + --> TIME: 2024-06-25 12:30:29 -- STEP: 456/591 -- GLOBAL_STEP: 35325 + | > loss_disc: 2.5753092765808105 (2.625971399901204) + | > loss_disc_real_0: 0.1539028137922287 (0.19838717140322715) + | > loss_disc_real_1: 0.22071221470832825 (0.22283286150348813) + | > loss_disc_real_2: 0.22440512478351593 (0.2315309686226803) + | > loss_disc_real_3: 0.2149743139743805 (0.23332483461943634) + | > loss_disc_real_4: 0.20980626344680786 (0.2087220237181897) + | > loss_disc_real_5: 0.23241573572158813 (0.238094547404009) + | > loss_0: 2.5753092765808105 (2.625971399901204) + | > grad_norm_0: tensor(26.9998, device='cuda:0') (tensor(27.8609, device='cuda:0')) + | > loss_gen: 2.1520183086395264 (2.0980982137353794) + | > loss_kl: 2.4561588764190674 (2.494534054869099) + | > loss_feat: 4.446382522583008 (4.5277634320551865) + | > loss_mel: 19.94761848449707 (20.49738853437859) + | > loss_duration: 1.3400273323059082 (1.3390373971901433) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.34220314025879 (30.95682166333784) + | > grad_norm_1: tensor(1689.1709, device='cuda:0') (tensor(1198.4893, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 0.998 (0.893292227857991) + | > loader_time: 0.0154 (0.014012114520658522) + + + --> TIME: 2024-06-25 12:30:56 -- STEP: 481/591 -- GLOBAL_STEP: 35350 + | > loss_disc: 2.65086030960083 (2.625603756636937) + | > loss_disc_real_0: 0.22632718086242676 (0.19816633905603584) + | > loss_disc_real_1: 0.17379413545131683 (0.22276009709076675) + | > loss_disc_real_2: 0.19279441237449646 (0.23145024822308466) + | > loss_disc_real_3: 0.2544134557247162 (0.23328997767888582) + | > loss_disc_real_4: 0.21878181397914886 (0.208882101081513) + | > loss_disc_real_5: 0.2528928518295288 (0.23801805097571033) + | > loss_0: 2.65086030960083 (2.625603756636937) + | > grad_norm_0: tensor(26.6959, device='cuda:0') (tensor(28.0564, device='cuda:0')) + | > loss_gen: 2.2893218994140625 (2.0980999516845986) + | > loss_kl: 2.4942660331726074 (2.4933073010117486) + | > loss_feat: 4.473189353942871 (4.521913600314925) + | > loss_mel: 20.524091720581055 (20.486341456613523) + | > loss_duration: 1.357309103012085 (1.3405273320025555) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.1381778717041 (30.94018966294128) + | > grad_norm_1: tensor(1335.5707, device='cuda:0') (tensor(1199.1260, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 1.0687 (0.9002350635687177) + | > loader_time: 0.0207 (0.01430362128418349) + + + --> TIME: 2024-06-25 12:31:24 -- STEP: 506/591 -- GLOBAL_STEP: 35375 + | > loss_disc: 2.588006019592285 (2.6262173327532663) + | > loss_disc_real_0: 0.14968273043632507 (0.19804469794918425) + | > loss_disc_real_1: 0.22850927710533142 (0.22301559921781064) + | > loss_disc_real_2: 0.23666764795780182 (0.23154647265498346) + | > loss_disc_real_3: 0.22712861001491547 (0.2333243264275577) + | > loss_disc_real_4: 0.21284139156341553 (0.20912715395095313) + | > loss_disc_real_5: 0.2369280457496643 (0.23804392751026532) + | > loss_0: 2.588006019592285 (2.6262173327532663) + | > grad_norm_0: tensor(30.7104, device='cuda:0') (tensor(27.9224, device='cuda:0')) + | > loss_gen: 2.1000406742095947 (2.0957475500144502) + | > loss_kl: 2.5316967964172363 (2.492482369125124) + | > loss_feat: 3.970287322998047 (4.509223005988379) + | > loss_mel: 19.706615447998047 (20.46299411562592) + | > loss_duration: 1.375672698020935 (1.34238022471605) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.68431282043457 (30.902827281725738) + | > grad_norm_1: tensor(1271.6179, device='cuda:0') (tensor(1190.7255, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 1.1651 (0.9097103384643673) + | > loader_time: 0.021 (0.014625546018125508) + + + --> TIME: 2024-06-25 12:31:52 -- STEP: 531/591 -- GLOBAL_STEP: 35400 + | > loss_disc: 2.7346439361572266 (2.627562268754631) + | > loss_disc_real_0: 0.2196895182132721 (0.19825717597750844) + | > loss_disc_real_1: 0.23643681406974792 (0.2230808475877381) + | > loss_disc_real_2: 0.25487929582595825 (0.2316214882272336) + | > loss_disc_real_3: 0.21120572090148926 (0.23343496563407662) + | > loss_disc_real_4: 0.22700338065624237 (0.20917639386070197) + | > loss_disc_real_5: 0.24867896735668182 (0.23811791486434794) + | > loss_0: 2.7346439361572266 (2.627562268754631) + | > grad_norm_0: tensor(8.3679, device='cuda:0') (tensor(27.9772, device='cuda:0')) + | > loss_gen: 1.911130666732788 (2.093810777206205) + | > loss_kl: 2.457242012023926 (2.489799573596587) + | > loss_feat: 3.918318033218384 (4.500051666787786) + | > loss_mel: 19.52642059326172 (20.448260452787757) + | > loss_duration: 1.411415457725525 (1.344082097549222) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.22452735900879 (30.876004572866552) + | > grad_norm_1: tensor(542.5264, device='cuda:0') (tensor(1186.3164, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 1.1191 (0.9191999053775476) + | > loader_time: 0.0256 (0.014958597621450763) + + + --> TIME: 2024-06-25 12:32:23 -- STEP: 556/591 -- GLOBAL_STEP: 35425 + | > loss_disc: 2.539811134338379 (2.626728798416878) + | > loss_disc_real_0: 0.28712883591651917 (0.19827282180835448) + | > loss_disc_real_1: 0.24905049800872803 (0.22314485280419424) + | > loss_disc_real_2: 0.2832246422767639 (0.23165839993589216) + | > loss_disc_real_3: 0.22620515525341034 (0.23347608863020972) + | > loss_disc_real_4: 0.2239747941493988 (0.2088657912376115) + | > loss_disc_real_5: 0.25054872035980225 (0.23805369597544773) + | > loss_0: 2.539811134338379 (2.626728798416878) + | > grad_norm_0: tensor(32.8468, device='cuda:0') (tensor(28.1069, device='cuda:0')) + | > loss_gen: 2.2149012088775635 (2.0967960953712463) + | > loss_kl: 2.4116179943084717 (2.4876049298176657) + | > loss_feat: 4.878597736358643 (4.498405839470648) + | > loss_mel: 20.766437530517578 (20.437751447554117) + | > loss_duration: 1.401963710784912 (1.3458942475078766) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.67351722717285 (30.866452567011333) + | > grad_norm_1: tensor(489.6609, device='cuda:0') (tensor(1177.8600, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 1.1657 (0.9314797644992524) + | > loader_time: 0.0211 (0.015340093228456786) + + + --> TIME: 2024-06-25 12:32:54 -- STEP: 581/591 -- GLOBAL_STEP: 35450 + | > loss_disc: 2.697871685028076 (2.628265590963182) + | > loss_disc_real_0: 0.2274315357208252 (0.19816730686955525) + | > loss_disc_real_1: 0.21306367218494415 (0.22352126807538655) + | > loss_disc_real_2: 0.21311861276626587 (0.2316043963223029) + | > loss_disc_real_3: 0.21331384778022766 (0.23346602632030972) + | > loss_disc_real_4: 0.27080684900283813 (0.2093086594302149) + | > loss_disc_real_5: 0.2693999409675598 (0.2381799857351743) + | > loss_0: 2.697871685028076 (2.628265590963182) + | > grad_norm_0: tensor(13.5595, device='cuda:0') (tensor(27.8639, device='cuda:0')) + | > loss_gen: 1.8684523105621338 (2.096209154687116) + | > loss_kl: 2.3555240631103516 (2.486114768686475) + | > loss_feat: 3.7254433631896973 (4.4890709953340915) + | > loss_mel: 20.05699348449707 (20.432035587330496) + | > loss_duration: 1.3870384693145752 (1.3475779272397899) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.393451690673828 (30.851008439843707) + | > grad_norm_1: tensor(516.8267, device='cuda:0') (tensor(1155.2983, device='cuda:0')) + | > current_lr_0: 0.00019853033419836382 + | > current_lr_1: 0.00019853033419836382 + | > step_time: 1.2758 (0.9443158331918632) + | > loader_time: 0.0246 (0.01574502549360213) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0035573776732099817 (-1.2377475170377052e-06) + | > avg_loss_disc: 2.6759740291757788 (+0.017132388784529606) + | > avg_loss_disc_real_0: 0.2095778150127289 (+0.039852297686515964) + | > avg_loss_disc_real_1: 0.194992573654398 (-0.010858091585179586) + | > avg_loss_disc_real_2: 0.22124761945389687 (-0.004757161787215691) + | > avg_loss_disc_real_3: 0.25174296441230376 (+0.019591213857874046) + | > avg_loss_disc_real_4: 0.2641642004885572 (+0.08314544390490716) + | > avg_loss_disc_real_5: 0.24062587035463212 (+0.005345652078060398) + | > avg_loss_0: 2.6759740291757788 (+0.017132388784529606) + | > avg_loss_gen: 1.9444745094218152 (+0.13324312199937527) + | > avg_loss_kl: 2.5028895114330534 (-0.04358976445299545) + | > avg_loss_feat: 4.107786001043118 (-0.11879814939295841) + | > avg_loss_mel: 20.293853962675048 (+0.0035095620662488614) + | > avg_loss_duration: 1.3586861148793647 (+0.0007065940410533322) + | > avg_loss_1: 30.207690178079808 (-0.02492868139388804) + + + > EPOCH: 61/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 12:33:15)  + + --> TIME: 2024-06-25 12:33:28 -- STEP: 15/591 -- GLOBAL_STEP: 35475 + | > loss_disc: 2.7016923427581787 (2.63491047223409) + | > loss_disc_real_0: 0.2525377869606018 (0.19462035497029623) + | > loss_disc_real_1: 0.22253386676311493 (0.21947471400101978) + | > loss_disc_real_2: 0.22771324217319489 (0.23269802729288738) + | > loss_disc_real_3: 0.21640312671661377 (0.2289203921953837) + | > loss_disc_real_4: 0.2277204543352127 (0.236855482061704) + | > loss_disc_real_5: 0.21375314891338348 (0.2346730430920919) + | > loss_0: 2.7016923427581787 (2.63491047223409) + | > grad_norm_0: tensor(15.2874, device='cuda:0') (tensor(10.2198, device='cuda:0')) + | > loss_gen: 1.9797399044036865 (2.044012769063314) + | > loss_kl: 2.4737560749053955 (2.5949135144551594) + | > loss_feat: 4.680934906005859 (4.613211663564046) + | > loss_mel: 20.27358627319336 (20.516416931152342) + | > loss_duration: 1.308646321296692 (1.306637692451477) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.716663360595703 (31.07519251505534) + | > grad_norm_1: tensor(409.3453, device='cuda:0') (tensor(377.0081, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.7798 (0.7692087332407633) + | > loader_time: 0.01 (0.008935372034708658) + + + --> TIME: 2024-06-25 12:33:48 -- STEP: 40/591 -- GLOBAL_STEP: 35500 + | > loss_disc: 2.61557674407959 (2.6423436939716334) + | > loss_disc_real_0: 0.25080347061157227 (0.19741824194788932) + | > loss_disc_real_1: 0.2215615212917328 (0.22030305601656436) + | > loss_disc_real_2: 0.21120350062847137 (0.23130449540913106) + | > loss_disc_real_3: 0.22108599543571472 (0.2326569989323616) + | > loss_disc_real_4: 0.20383326709270477 (0.22930721305310725) + | > loss_disc_real_5: 0.24978074431419373 (0.2366767432540655) + | > loss_0: 2.61557674407959 (2.6423436939716334) + | > grad_norm_0: tensor(14.1138, device='cuda:0') (tensor(15.7711, device='cuda:0')) + | > loss_gen: 1.98758864402771 (2.04084151983261) + | > loss_kl: 2.3418843746185303 (2.5581895411014557) + | > loss_feat: 4.485363006591797 (4.5360104799270635) + | > loss_mel: 20.333232879638672 (20.526781272888183) + | > loss_duration: 1.3650062084197998 (1.3167588740587237) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.51307487487793 (30.978581762313844) + | > grad_norm_1: tensor(61.7457, device='cuda:0') (tensor(690.5298, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8012 (0.7836566925048827) + | > loader_time: 0.0096 (0.009459036588668823) + + + --> TIME: 2024-06-25 12:34:09 -- STEP: 65/591 -- GLOBAL_STEP: 35525 + | > loss_disc: 2.8387184143066406 (2.6314223216130177) + | > loss_disc_real_0: 0.2143135517835617 (0.1988323369851479) + | > loss_disc_real_1: 0.25332877039909363 (0.22032657540761508) + | > loss_disc_real_2: 0.23411241173744202 (0.22982472593967732) + | > loss_disc_real_3: 0.24796746671199799 (0.2329373265688236) + | > loss_disc_real_4: 0.18757383525371552 (0.21729984100048358) + | > loss_disc_real_5: 0.2320379763841629 (0.2369479553057597) + | > loss_0: 2.8387184143066406 (2.6314223216130177) + | > grad_norm_0: tensor(21.5951, device='cuda:0') (tensor(20.4434, device='cuda:0')) + | > loss_gen: 1.963796615600586 (2.085276335936326) + | > loss_kl: 2.368037223815918 (2.5351413286649267) + | > loss_feat: 4.591786861419678 (4.625067314734825) + | > loss_mel: 21.33661460876465 (20.582399485661433) + | > loss_duration: 1.3457468748092651 (1.325385796106779) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.605981826782227 (31.153270310621995) + | > grad_norm_1: tensor(271.6046, device='cuda:0') (tensor(849.8691, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8173 (0.7948651753939114) + | > loader_time: 0.0101 (0.00987071624168983) + + + --> TIME: 2024-06-25 12:34:30 -- STEP: 90/591 -- GLOBAL_STEP: 35550 + | > loss_disc: 2.6254923343658447 (2.6262660370932687) + | > loss_disc_real_0: 0.15411967039108276 (0.19867893722322252) + | > loss_disc_real_1: 0.19854310154914856 (0.22108593318197461) + | > loss_disc_real_2: 0.21387560665607452 (0.22972798198461533) + | > loss_disc_real_3: 0.2144617736339569 (0.23298689723014832) + | > loss_disc_real_4: 0.25346535444259644 (0.21239606224828297) + | > loss_disc_real_5: 0.24205456674098969 (0.23802964720461103) + | > loss_0: 2.6254923343658447 (2.6262660370932687) + | > grad_norm_0: tensor(36.4829, device='cuda:0') (tensor(22.8422, device='cuda:0')) + | > loss_gen: 2.256669759750366 (2.1005905614958875) + | > loss_kl: 2.4376444816589355 (2.522891743977865) + | > loss_feat: 4.4544196128845215 (4.625733968946668) + | > loss_mel: 20.13728904724121 (20.555050892300077) + | > loss_duration: 1.3501629829406738 (1.3321353753407794) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.636186599731445 (31.136402553982204) + | > grad_norm_1: tensor(1729.1785, device='cuda:0') (tensor(968.0449, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8273 (0.8027521795696683) + | > loader_time: 0.0109 (0.010074687004089356) + + + --> TIME: 2024-06-25 12:34:51 -- STEP: 115/591 -- GLOBAL_STEP: 35575 + | > loss_disc: 2.613593816757202 (2.626885101069575) + | > loss_disc_real_0: 0.15544304251670837 (0.19873281762651776) + | > loss_disc_real_1: 0.21095015108585358 (0.22160714268684387) + | > loss_disc_real_2: 0.22848908603191376 (0.23070212654445482) + | > loss_disc_real_3: 0.23497511446475983 (0.2332484886698101) + | > loss_disc_real_4: 0.23207662999629974 (0.21133868266706882) + | > loss_disc_real_5: 0.26283955574035645 (0.2384204363045485) + | > loss_0: 2.613593816757202 (2.626885101069575) + | > grad_norm_0: tensor(30.0359, device='cuda:0') (tensor(24.1164, device='cuda:0')) + | > loss_gen: 2.17142915725708 (2.1047040897866958) + | > loss_kl: 2.5415375232696533 (2.5162154591601826) + | > loss_feat: 4.710832595825195 (4.626059071914011) + | > loss_mel: 21.047435760498047 (20.536387551349144) + | > loss_duration: 1.3364031314849854 (1.3346260682396267) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.807636260986328 (31.117992268437924) + | > grad_norm_1: tensor(1197.9871, device='cuda:0') (tensor(1018.4844, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8508 (0.8097005719723909) + | > loader_time: 0.0107 (0.010281890371571416) + + + --> TIME: 2024-06-25 12:35:13 -- STEP: 140/591 -- GLOBAL_STEP: 35600 + | > loss_disc: 2.5774803161621094 (2.625928272519793) + | > loss_disc_real_0: 0.17176416516304016 (0.19946997091174126) + | > loss_disc_real_1: 0.2124403417110443 (0.2219028999762876) + | > loss_disc_real_2: 0.236912339925766 (0.23092293121985027) + | > loss_disc_real_3: 0.21993401646614075 (0.23327745454651969) + | > loss_disc_real_4: 0.1747666448354721 (0.2100642516144684) + | > loss_disc_real_5: 0.23857493698596954 (0.23788512572646142) + | > loss_0: 2.5774803161621094 (2.625928272519793) + | > grad_norm_0: tensor(30.2583, device='cuda:0') (tensor(24.8819, device='cuda:0')) + | > loss_gen: 2.047882080078125 (2.1062119824545724) + | > loss_kl: 2.599560499191284 (2.5147236704826352) + | > loss_feat: 4.646108150482178 (4.621298013414658) + | > loss_mel: 20.46243667602539 (20.520906026022775) + | > loss_duration: 1.3209500312805176 (1.3346794341291701) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.076936721801758 (31.097819137573243) + | > grad_norm_1: tensor(1306.2654, device='cuda:0') (tensor(1049.7866, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8469 (0.8165103367396763) + | > loader_time: 0.0112 (0.010531987462724957) + + + --> TIME: 2024-06-25 12:35:35 -- STEP: 165/591 -- GLOBAL_STEP: 35625 + | > loss_disc: 2.545982837677002 (2.619239531141339) + | > loss_disc_real_0: 0.15520434081554413 (0.19823344096992956) + | > loss_disc_real_1: 0.19717249274253845 (0.22198356317751336) + | > loss_disc_real_2: 0.1836821436882019 (0.23058852011507208) + | > loss_disc_real_3: 0.1667477786540985 (0.2331300889000748) + | > loss_disc_real_4: 0.16550064086914062 (0.2069748785459633) + | > loss_disc_real_5: 0.2326391041278839 (0.23779816392696265) + | > loss_0: 2.545982837677002 (2.619239531141339) + | > grad_norm_0: tensor(26.6547, device='cuda:0') (tensor(25.5676, device='cuda:0')) + | > loss_gen: 2.248208999633789 (2.1168066125927547) + | > loss_kl: 2.563230514526367 (2.512475987636681) + | > loss_feat: 5.069952964782715 (4.640730574636749) + | > loss_mel: 20.830108642578125 (20.54210311427261) + | > loss_duration: 1.3090616464614868 (1.3332379557869654) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.02056121826172 (31.145354288274593) + | > grad_norm_1: tensor(1449.0776, device='cuda:0') (tensor(1065.5864, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8612 (0.8235111843455921) + | > loader_time: 0.0133 (0.010779379353378759) + + + --> TIME: 2024-06-25 12:35:57 -- STEP: 190/591 -- GLOBAL_STEP: 35650 + | > loss_disc: 2.5782337188720703 (2.6205051509957555) + | > loss_disc_real_0: 0.17503103613853455 (0.19832421917664378) + | > loss_disc_real_1: 0.23057487607002258 (0.22172897395334745) + | > loss_disc_real_2: 0.2307450920343399 (0.23088044932014065) + | > loss_disc_real_3: 0.19910454750061035 (0.23334171419080935) + | > loss_disc_real_4: 0.20599231123924255 (0.2074660805495161) + | > loss_disc_real_5: 0.25651854276657104 (0.23797831801991715) + | > loss_0: 2.5782337188720703 (2.6205051509957555) + | > grad_norm_0: tensor(24.1781, device='cuda:0') (tensor(25.2995, device='cuda:0')) + | > loss_gen: 2.153944492340088 (2.11126001822321) + | > loss_kl: 2.661686897277832 (2.5121379739359795) + | > loss_feat: 4.60170316696167 (4.6260015763734526) + | > loss_mel: 20.09398651123047 (20.517891151026678) + | > loss_duration: 1.3538997173309326 (1.3315363469876742) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.865219116210938 (31.098827070938913) + | > grad_norm_1: tensor(1426.4563, device='cuda:0') (tensor(1058.5470, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8663 (0.8302703054327714) + | > loader_time: 0.0135 (0.010968762949893346) + + + --> TIME: 2024-06-25 12:36:19 -- STEP: 215/591 -- GLOBAL_STEP: 35675 + | > loss_disc: 2.7086853981018066 (2.6263980676961487) + | > loss_disc_real_0: 0.2686029374599457 (0.19955763439106386) + | > loss_disc_real_1: 0.2630457878112793 (0.2220684313496878) + | > loss_disc_real_2: 0.25546374917030334 (0.23088187658509543) + | > loss_disc_real_3: 0.22891759872436523 (0.23337646695070488) + | > loss_disc_real_4: 0.23614458739757538 (0.20969197382760596) + | > loss_disc_real_5: 0.23978732526302338 (0.23782699627931728) + | > loss_0: 2.7086853981018066 (2.6263980676961487) + | > grad_norm_0: tensor(11.0449, device='cuda:0') (tensor(24.8653, device='cuda:0')) + | > loss_gen: 1.9147692918777466 (2.0990535043006715) + | > loss_kl: 2.404294013977051 (2.5111211521680956) + | > loss_feat: 4.400611877441406 (4.594130417358044) + | > loss_mel: 20.150373458862305 (20.49286001338515) + | > loss_duration: 1.3700478076934814 (1.3308069888935532) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.24009895324707 (31.027972057253816) + | > grad_norm_1: tensor(115.0756, device='cuda:0') (tensor(1034.3126, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8632 (0.8357692785041276) + | > loader_time: 0.0146 (0.011220224513563996) + + + --> TIME: 2024-06-25 12:36:42 -- STEP: 240/591 -- GLOBAL_STEP: 35700 + | > loss_disc: 2.6211161613464355 (2.6264717370271673) + | > loss_disc_real_0: 0.1965179741382599 (0.199282184895128) + | > loss_disc_real_1: 0.22414755821228027 (0.22158201144387324) + | > loss_disc_real_2: 0.2495744824409485 (0.23064019344747066) + | > loss_disc_real_3: 0.2398735135793686 (0.2331151531388362) + | > loss_disc_real_4: 0.1939573436975479 (0.21117855316648876) + | > loss_disc_real_5: 0.2396635264158249 (0.23775831094632546) + | > loss_0: 2.6211161613464355 (2.6264717370271673) + | > grad_norm_0: tensor(29.1834, device='cuda:0') (tensor(24.3245, device='cuda:0')) + | > loss_gen: 2.128726005554199 (2.09366121192773) + | > loss_kl: 2.534156084060669 (2.5096724927425376) + | > loss_feat: 4.8218560218811035 (4.583978872497878) + | > loss_mel: 20.678730010986328 (20.5088432153066) + | > loss_duration: 1.344957947731018 (1.330564377705256) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.508424758911133 (31.02672016620636) + | > grad_norm_1: tensor(1464.7610, device='cuda:0') (tensor(1011.6270, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.8771 (0.8409606774648031) + | > loader_time: 0.0129 (0.011516507466634112) + + + --> TIME: 2024-06-25 12:37:05 -- STEP: 265/591 -- GLOBAL_STEP: 35725 + | > loss_disc: 2.553813934326172 (2.6251211067415623) + | > loss_disc_real_0: 0.18259359896183014 (0.19884272782870058) + | > loss_disc_real_1: 0.21190369129180908 (0.2218014621509696) + | > loss_disc_real_2: 0.24046072363853455 (0.2309354432348935) + | > loss_disc_real_3: 0.2845550775527954 (0.23333637770616783) + | > loss_disc_real_4: 0.1844152808189392 (0.21020774745716234) + | > loss_disc_real_5: 0.23942193388938904 (0.23792464035861896) + | > loss_0: 2.553813934326172 (2.6251211067415623) + | > grad_norm_0: tensor(25.4593, device='cuda:0') (tensor(24.5166, device='cuda:0')) + | > loss_gen: 2.380964756011963 (2.0985717584502006) + | > loss_kl: 2.465545177459717 (2.5077600029279594) + | > loss_feat: 5.063360214233398 (4.589614793489565) + | > loss_mel: 19.699148178100586 (20.504478058725034) + | > loss_duration: 1.3019931316375732 (1.3305475981730335) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.9110107421875 (31.030972196471016) + | > grad_norm_1: tensor(1537.7332, device='cuda:0') (tensor(1032.7382, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.9573 (0.8472784438223208) + | > loader_time: 0.0132 (0.011818772441935986) + + + --> TIME: 2024-06-25 12:37:28 -- STEP: 290/591 -- GLOBAL_STEP: 35750 + | > loss_disc: 2.6218249797821045 (2.6254061731798886) + | > loss_disc_real_0: 0.1799635887145996 (0.1987679371289138) + | > loss_disc_real_1: 0.18269404768943787 (0.22205471226881285) + | > loss_disc_real_2: 0.22056710720062256 (0.23079150051906192) + | > loss_disc_real_3: 0.21452057361602783 (0.23331987148728864) + | > loss_disc_real_4: 0.18501845002174377 (0.209231717473474) + | > loss_disc_real_5: 0.19561785459518433 (0.23796665647934223) + | > loss_0: 2.6218249797821045 (2.6254061731798886) + | > grad_norm_0: tensor(9.8191, device='cuda:0') (tensor(24.7046, device='cuda:0')) + | > loss_gen: 2.0506622791290283 (2.1016048752028347) + | > loss_kl: 2.3726024627685547 (2.5055075086396306) + | > loss_feat: 4.538580417633057 (4.590973378049919) + | > loss_mel: 20.609750747680664 (20.508935540297937) + | > loss_duration: 1.2824523448944092 (1.331439301885408) + | > amp_scaler: 128.0 (67.75172413793105) + | > loss_1: 30.854047775268555 (31.038460586810935) + | > grad_norm_1: tensor(509.1025, device='cuda:0') (tensor(1025.5077, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.9292 (0.8526795666793298) + | > loader_time: 0.0162 (0.012097867603959709) + + + --> TIME: 2024-06-25 12:37:52 -- STEP: 315/591 -- GLOBAL_STEP: 35775 + | > loss_disc: 2.732537031173706 (2.6243804462372284) + | > loss_disc_real_0: 0.20843511819839478 (0.19850379093771894) + | > loss_disc_real_1: 0.25889453291893005 (0.22232026192877022) + | > loss_disc_real_2: 0.23074603080749512 (0.23111458572130356) + | > loss_disc_real_3: 0.20984646677970886 (0.2332668879675487) + | > loss_disc_real_4: 0.25266751646995544 (0.20862117293808197) + | > loss_disc_real_5: 0.2152104377746582 (0.23766249437180775) + | > loss_0: 2.732537031173706 (2.6243804462372284) + | > grad_norm_0: tensor(65.2356, device='cuda:0') (tensor(25.2770, device='cuda:0')) + | > loss_gen: 2.2716076374053955 (2.109374904632566) + | > loss_kl: 2.4094879627227783 (2.5044739049578455) + | > loss_feat: 4.53499698638916 (4.604125486858313) + | > loss_mel: 20.293533325195312 (20.519604328700474) + | > loss_duration: 1.340332269668579 (1.3322157840880147) + | > amp_scaler: 64.0 (69.07936507936512) + | > loss_1: 30.849958419799805 (31.069794415670728) + | > grad_norm_1: tensor(1148.1661, device='cuda:0') (tensor(1019.8531, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.9326 (0.8586015943496946) + | > loader_time: 0.0138 (0.012322321392240982) + + + --> TIME: 2024-06-25 12:38:16 -- STEP: 340/591 -- GLOBAL_STEP: 35800 + | > loss_disc: 2.6851372718811035 (2.6261555510408714) + | > loss_disc_real_0: 0.16165749728679657 (0.19841264159802133) + | > loss_disc_real_1: 0.21785250306129456 (0.2232883714139461) + | > loss_disc_real_2: 0.2247447520494461 (0.2314244267256821) + | > loss_disc_real_3: 0.261157363653183 (0.23340025544166565) + | > loss_disc_real_4: 0.22955630719661713 (0.2084034570657155) + | > loss_disc_real_5: 0.26031363010406494 (0.2379393249311868) + | > loss_0: 2.6851372718811035 (2.6261555510408714) + | > grad_norm_0: tensor(14.6274, device='cuda:0') (tensor(25.4577, device='cuda:0')) + | > loss_gen: 2.113034725189209 (2.1101308913791854) + | > loss_kl: 2.4908676147460938 (2.5037452487384546) + | > loss_feat: 4.323987007141113 (4.596569161555354) + | > loss_mel: 20.382230758666992 (20.525326442718505) + | > loss_duration: 1.3581773042678833 (1.3335818294216613) + | > amp_scaler: 64.0 (68.7058823529412) + | > loss_1: 30.668298721313477 (31.069353569255156) + | > grad_norm_1: tensor(472.1284, device='cuda:0') (tensor(1019.9869, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 1.0002 (0.8637922111679526) + | > loader_time: 0.0161 (0.012585848920485553) + + + --> TIME: 2024-06-25 12:38:40 -- STEP: 365/591 -- GLOBAL_STEP: 35825 + | > loss_disc: 2.6871893405914307 (2.6298304590460346) + | > loss_disc_real_0: 0.20469845831394196 (0.19883580642612017) + | > loss_disc_real_1: 0.21486052870750427 (0.2237142151757462) + | > loss_disc_real_2: 0.216019868850708 (0.23152858386301015) + | > loss_disc_real_3: 0.2487819790840149 (0.23334885202858546) + | > loss_disc_real_4: 0.227695032954216 (0.20981181147572112) + | > loss_disc_real_5: 0.24024862051010132 (0.23785088556270076) + | > loss_0: 2.6871893405914307 (2.6298304590460346) + | > grad_norm_0: tensor(26.2117, device='cuda:0') (tensor(25.0864, device='cuda:0')) + | > loss_gen: 2.004629611968994 (2.1020068070659863) + | > loss_kl: 2.5244076251983643 (2.5025516640650056) + | > loss_feat: 4.389902114868164 (4.572990011842293) + | > loss_mel: 20.9515380859375 (20.514881149710053) + | > loss_duration: 1.3462426662445068 (1.3351811278356271) + | > amp_scaler: 64.0 (68.38356164383569) + | > loss_1: 31.216720581054688 (31.027610763131754) + | > grad_norm_1: tensor(858.7375, device='cuda:0') (tensor(1001.0801, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.9387 (0.869783511880326) + | > loader_time: 0.015 (0.012854523854712918) + + + --> TIME: 2024-06-25 12:39:05 -- STEP: 390/591 -- GLOBAL_STEP: 35850 + | > loss_disc: 2.5979292392730713 (2.6309241942870276) + | > loss_disc_real_0: 0.15758249163627625 (0.1986471826449419) + | > loss_disc_real_1: 0.2359413206577301 (0.223729154582207) + | > loss_disc_real_2: 0.21012602746486664 (0.23176885277797013) + | > loss_disc_real_3: 0.23313400149345398 (0.23334648066606278) + | > loss_disc_real_4: 0.2156623899936676 (0.2106004378161369) + | > loss_disc_real_5: 0.23264752328395844 (0.237841680034613) + | > loss_0: 2.5979292392730713 (2.6309241942870276) + | > grad_norm_0: tensor(28.6273, device='cuda:0') (tensor(24.9004, device='cuda:0')) + | > loss_gen: 2.0699045658111572 (2.096869543882515) + | > loss_kl: 2.6853411197662354 (2.503117502652679) + | > loss_feat: 4.498193264007568 (4.555968461281221) + | > loss_mel: 20.28286361694336 (20.506296157836932) + | > loss_duration: 1.3775901794433594 (1.3361868959206806) + | > amp_scaler: 64.0 (68.10256410256414) + | > loss_1: 30.91389274597168 (30.998438536815154) + | > grad_norm_1: tensor(1332.6903, device='cuda:0') (tensor(996.5688, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.9564 (0.8768253503701626) + | > loader_time: 0.0153 (0.01312877031473013) + + + --> TIME: 2024-06-25 12:39:30 -- STEP: 415/591 -- GLOBAL_STEP: 35875 + | > loss_disc: 2.6841869354248047 (2.632790479889832) + | > loss_disc_real_0: 0.10998934507369995 (0.1989173156310277) + | > loss_disc_real_1: 0.22093138098716736 (0.2236291087176426) + | > loss_disc_real_2: 0.26309022307395935 (0.2319177915891969) + | > loss_disc_real_3: 0.25822803378105164 (0.2332824385668858) + | > loss_disc_real_4: 0.22763702273368835 (0.21103500108043832) + | > loss_disc_real_5: 0.2747041881084442 (0.23796810967376433) + | > loss_0: 2.6841869354248047 (2.632790479889832) + | > grad_norm_0: tensor(28.3763, device='cuda:0') (tensor(25.0483, device='cuda:0')) + | > loss_gen: 2.054743766784668 (2.0936882409704727) + | > loss_kl: 2.3637077808380127 (2.5026988718883056) + | > loss_feat: 4.001744747161865 (4.5423020121563) + | > loss_mel: 19.807727813720703 (20.490501969119173) + | > loss_duration: 1.3543472290039062 (1.3371839704283754) + | > amp_scaler: 64.0 (67.85542168674705) + | > loss_1: 29.582271575927734 (30.966375052210797) + | > grad_norm_1: tensor(1071.8649, device='cuda:0') (tensor(996.4044, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 1.0157 (0.8832653895918146) + | > loader_time: 0.0195 (0.013414941925600351) + + + --> TIME: 2024-06-25 12:39:56 -- STEP: 440/591 -- GLOBAL_STEP: 35900 + | > loss_disc: 2.693171739578247 (2.6319873690605133) + | > loss_disc_real_0: 0.20890235900878906 (0.19870167801325975) + | > loss_disc_real_1: 0.20080065727233887 (0.2234953454949639) + | > loss_disc_real_2: 0.23553793132305145 (0.2317745625634085) + | > loss_disc_real_3: 0.21950899064540863 (0.23319608321921392) + | > loss_disc_real_4: 0.18278957903385162 (0.2109056946736845) + | > loss_disc_real_5: 0.2246689796447754 (0.2380662896416404) + | > loss_0: 2.693171739578247 (2.6319873690605133) + | > grad_norm_0: tensor(31.5794, device='cuda:0') (tensor(25.0608, device='cuda:0')) + | > loss_gen: 1.9875274896621704 (2.091990599849005) + | > loss_kl: 2.5465822219848633 (2.500128347223451) + | > loss_feat: 4.352366924285889 (4.532817670432009) + | > loss_mel: 20.242095947265625 (20.481311802430607) + | > loss_duration: 1.3376072645187378 (1.3381678253412252) + | > amp_scaler: 64.0 (67.63636363636368) + | > loss_1: 30.46617889404297 (30.944416232542558) + | > grad_norm_1: tensor(1434.3004, device='cuda:0') (tensor(997.8322, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 1.0758 (0.8901586137034676) + | > loader_time: 0.0237 (0.013717307285829023) + + + --> TIME: 2024-06-25 12:40:22 -- STEP: 465/591 -- GLOBAL_STEP: 35925 + | > loss_disc: 2.641352653503418 (2.6339803408550932) + | > loss_disc_real_0: 0.2918374538421631 (0.19873994390169783) + | > loss_disc_real_1: 0.24072644114494324 (0.22354023975710713) + | > loss_disc_real_2: 0.21620365977287292 (0.23187241602328515) + | > loss_disc_real_3: 0.23731370270252228 (0.2335865548861924) + | > loss_disc_real_4: 0.19933810830116272 (0.21161527987769854) + | > loss_disc_real_5: 0.24017316102981567 (0.23810328866845817) + | > loss_0: 2.641352653503418 (2.6339803408550932) + | > grad_norm_0: tensor(25.0236, device='cuda:0') (tensor(25.1048, device='cuda:0')) + | > loss_gen: 1.8028357028961182 (2.0894480115623866) + | > loss_kl: 2.3994500637054443 (2.4976070762962386) + | > loss_feat: 4.021579742431641 (4.517742200051589) + | > loss_mel: 19.504289627075195 (20.46093738514892) + | > loss_duration: 1.3670790195465088 (1.3392484198334402) + | > amp_scaler: 64.0 (67.44086021505382) + | > loss_1: 29.095233917236328 (30.90498306930706) + | > grad_norm_1: tensor(61.7440, device='cuda:0') (tensor(994.9669, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 1.0289 (0.8975422305445517) + | > loader_time: 0.0176 (0.013989286012546991) + + + --> TIME: 2024-06-25 12:40:49 -- STEP: 490/591 -- GLOBAL_STEP: 35950 + | > loss_disc: 2.7088277339935303 (2.633327399954501) + | > loss_disc_real_0: 0.14963652193546295 (0.19857674508678672) + | > loss_disc_real_1: 0.24560897052288055 (0.22350205639187168) + | > loss_disc_real_2: 0.23816026747226715 (0.2319654440393253) + | > loss_disc_real_3: 0.2533927261829376 (0.2337652648590049) + | > loss_disc_real_4: 0.1935778707265854 (0.21111907334047922) + | > loss_disc_real_5: 0.27353909611701965 (0.23806079708191813) + | > loss_0: 2.7088277339935303 (2.633327399954501) + | > grad_norm_0: tensor(26.1970, device='cuda:0') (tensor(25.3649, device='cuda:0')) + | > loss_gen: 1.8939571380615234 (2.0911448546818305) + | > loss_kl: 2.4523766040802 (2.4955001164455766) + | > loss_feat: 4.242772102355957 (4.5163818242598595) + | > loss_mel: 19.663105010986328 (20.451462737881425) + | > loss_duration: 1.3620461225509644 (1.3407123906271803) + | > amp_scaler: 64.0 (67.26530612244908) + | > loss_1: 29.614255905151367 (30.895201912704778) + | > grad_norm_1: tensor(1045.2340, device='cuda:0') (tensor(1001.2010, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 1.0486 (0.9050060870696087) + | > loader_time: 0.0191 (0.014262982290618273) + + + --> TIME: 2024-06-25 12:41:17 -- STEP: 515/591 -- GLOBAL_STEP: 35975 + | > loss_disc: 2.648329496383667 (2.6328745841979955) + | > loss_disc_real_0: 0.3209867775440216 (0.19891927774670057) + | > loss_disc_real_1: 0.23353268206119537 (0.22337063420744774) + | > loss_disc_real_2: 0.26498499512672424 (0.23190666989794056) + | > loss_disc_real_3: 0.21737995743751526 (0.233641906560046) + | > loss_disc_real_4: 0.21284371614456177 (0.21098977068384875) + | > loss_disc_real_5: 0.23847180604934692 (0.23792972327436057) + | > loss_0: 2.648329496383667 (2.6328745841979955) + | > grad_norm_0: tensor(35.1370, device='cuda:0') (tensor(25.6278, device='cuda:0')) + | > loss_gen: 1.9614790678024292 (2.0911983918217762) + | > loss_kl: 2.3564960956573486 (2.493408373490117) + | > loss_feat: 4.147181034088135 (4.509840228256674) + | > loss_mel: 19.77950668334961 (20.434616866621017) + | > loss_duration: 1.3794023990631104 (1.3424216203319221) + | > amp_scaler: 64.0 (67.10679611650494) + | > loss_1: 29.624065399169922 (30.871485456448156) + | > grad_norm_1: tensor(1467.5306, device='cuda:0') (tensor(1012.6879, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 1.1517 (0.9152629861553896) + | > loader_time: 0.0176 (0.014543943034792409) + + + --> TIME: 2024-06-25 12:41:48 -- STEP: 540/591 -- GLOBAL_STEP: 36000 + | > loss_disc: 2.5789692401885986 (2.633078576017307) + | > loss_disc_real_0: 0.24742823839187622 (0.1986989996499486) + | > loss_disc_real_1: 0.22515031695365906 (0.2232950937692766) + | > loss_disc_real_2: 0.2316916286945343 (0.23184680654494852) + | > loss_disc_real_3: 0.18502859771251678 (0.23359617356348925) + | > loss_disc_real_4: 0.2265213131904602 (0.21124410950751216) + | > loss_disc_real_5: 0.25804173946380615 (0.23807888994062387) + | > loss_0: 2.5789692401885986 (2.633078576017307) + | > grad_norm_0: tensor(29.6523, device='cuda:0') (tensor(25.5940, device='cuda:0')) + | > loss_gen: 2.0907464027404785 (2.0886963239422536) + | > loss_kl: 2.4541430473327637 (2.4909723343672536) + | > loss_feat: 4.22526741027832 (4.496397791968455) + | > loss_mel: 20.049640655517578 (20.416646706616454) + | > loss_duration: 1.3586406707763672 (1.3439206920288236) + | > amp_scaler: 64.0 (66.96296296296305) + | > loss_1: 30.178438186645508 (30.836633816471807) + | > grad_norm_1: tensor(1371.4755, device='cuda:0') (tensor(1011.2800, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 1.1944 (0.9276894008671795) + | > loader_time: 0.0263 (0.014859611458248562) + + + --> TIME: 2024-06-25 12:42:19 -- STEP: 565/591 -- GLOBAL_STEP: 36025 + | > loss_disc: 2.568706512451172 (2.6323863172953086) + | > loss_disc_real_0: 0.14947713911533356 (0.19843889957508162) + | > loss_disc_real_1: 0.1892712265253067 (0.22330907282576096) + | > loss_disc_real_2: 0.22733531892299652 (0.23190343227006693) + | > loss_disc_real_3: 0.2661730945110321 (0.23379526615670296) + | > loss_disc_real_4: 0.16015072166919708 (0.21063410770312874) + | > loss_disc_real_5: 0.23414097726345062 (0.23808104258722965) + | > loss_0: 2.568706512451172 (2.6323863172953086) + | > grad_norm_0: tensor(35.1167, device='cuda:0') (tensor(25.7311, device='cuda:0')) + | > loss_gen: 2.0924079418182373 (2.0891017424321796) + | > loss_kl: 2.3940367698669434 (2.4883856499089574) + | > loss_feat: 4.545780658721924 (4.491448559803249) + | > loss_mel: 20.360288619995117 (20.401391912139637) + | > loss_duration: 1.406294822692871 (1.3456581383679829) + | > amp_scaler: 64.0 (66.83185840707972) + | > loss_1: 30.798809051513672 (30.81598598176399) + | > grad_norm_1: tensor(1599.3857, device='cuda:0') (tensor(1016.1708, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 1.255 (0.9409074095498144) + | > loader_time: 0.0206 (0.015185326179571912) + + + --> TIME: 2024-06-25 12:42:51 -- STEP: 590/591 -- GLOBAL_STEP: 36050 + | > loss_disc: 2.6056087017059326 (2.6322922884407665) + | > loss_disc_real_0: 0.19323106110095978 (0.19814963929228868) + | > loss_disc_real_1: 0.2620261609554291 (0.22335113425881176) + | > loss_disc_real_2: 0.198685422539711 (0.23183935173487258) + | > loss_disc_real_3: 0.2104479968547821 (0.23371183003409435) + | > loss_disc_real_4: 0.1504872888326645 (0.21094254877860263) + | > loss_disc_real_5: 0.20444776117801666 (0.23797114807165276) + | > loss_0: 2.6056087017059326 (2.6322922884407665) + | > grad_norm_0: tensor(18.5058, device='cuda:0') (tensor(25.6536, device='cuda:0')) + | > loss_gen: 2.279712438583374 (2.0878750194937483) + | > loss_kl: 2.444777011871338 (2.4863417277901827) + | > loss_feat: 4.567137241363525 (4.479838026176068) + | > loss_mel: 21.694665908813477 (20.387486642093993) + | > loss_duration: 1.363678216934204 (1.3473231012538338) + | > amp_scaler: 64.0 (66.71186440677974) + | > loss_1: 32.349971771240234 (30.788864501047943) + | > grad_norm_1: tensor(1356.1282, device='cuda:0') (tensor(1018.3138, device='cuda:0')) + | > current_lr_0: 0.000198505517906589 + | > current_lr_1: 0.000198505517906589 + | > step_time: 0.4877 (0.9536909992411985) + | > loader_time: 0.007 (0.0155005208516525) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003718325432310713 (+0.00016094775910073132) + | > avg_loss_disc: 2.598975587398448 (-0.07699844177733084) + | > avg_loss_disc_real_0: 0.18225794222126615 (-0.02731987279146275) + | > avg_loss_disc_real_1: 0.22407270428982187 (+0.029080130635423862) + | > avg_loss_disc_real_2: 0.2539631974823932 (+0.03271557802849634) + | > avg_loss_disc_real_3: 0.33484000284621046 (+0.0830970384339067) + | > avg_loss_disc_real_4: 0.21093494112187244 (-0.05322925936668477) + | > avg_loss_disc_real_5: 0.23904113883667805 (-0.0015847315179540744) + | > avg_loss_0: 2.598975587398448 (-0.07699844177733084) + | > avg_loss_gen: 2.201869979817816 (+0.25739547039600086) + | > avg_loss_kl: 2.5356946346607607 (+0.032805123227707345) + | > avg_loss_feat: 4.4528165928860926 (+0.34503059184297413) + | > avg_loss_mel: 20.223011950229083 (-0.070842012445965) + | > avg_loss_duration: 1.3641191137597917 (+0.00543299888042692) + | > avg_loss_1: 30.77751236773552 (+0.5698221896557101) + + + > EPOCH: 62/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 12:43:01)  + + --> TIME: 2024-06-25 12:43:21 -- STEP: 24/591 -- GLOBAL_STEP: 36075 + | > loss_disc: 2.774653911590576 (2.6297079920768733) + | > loss_disc_real_0: 0.20855191349983215 (0.2039430346339941) + | > loss_disc_real_1: 0.2492350935935974 (0.22389044736822447) + | > loss_disc_real_2: 0.2716137170791626 (0.23092122872670492) + | > loss_disc_real_3: 0.28032881021499634 (0.23734966975947222) + | > loss_disc_real_4: 0.2592319846153259 (0.20360922068357468) + | > loss_disc_real_5: 0.22182828187942505 (0.24166236817836761) + | > loss_0: 2.774653911590576 (2.6297079920768733) + | > grad_norm_0: tensor(24.8860, device='cuda:0') (tensor(29.4553, device='cuda:0')) + | > loss_gen: 2.0803160667419434 (2.1797937005758286) + | > loss_kl: 2.616302490234375 (2.5855619112650547) + | > loss_feat: 4.360482692718506 (4.817242403825124) + | > loss_mel: 20.443906784057617 (20.64088781674703) + | > loss_duration: 1.3191497325897217 (1.311059003074964) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.820158004760742 (31.534544467926025) + | > grad_norm_1: tensor(1223.0635, device='cuda:0') (tensor(1141.1268, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.782 (0.7744606733322144) + | > loader_time: 0.0091 (0.009141415357589718) + + + --> TIME: 2024-06-25 12:43:42 -- STEP: 49/591 -- GLOBAL_STEP: 36100 + | > loss_disc: 2.511772632598877 (2.6312584341788767) + | > loss_disc_real_0: 0.29412704706192017 (0.19983612761205558) + | > loss_disc_real_1: 0.22744299471378326 (0.2245032726501932) + | > loss_disc_real_2: 0.2451593428850174 (0.23236472965503224) + | > loss_disc_real_3: 0.22571131587028503 (0.2344864059467705) + | > loss_disc_real_4: 0.17462152242660522 (0.2071128098332152) + | > loss_disc_real_5: 0.2155270278453827 (0.2398802857009732) + | > loss_0: 2.511772632598877 (2.6312584341788767) + | > grad_norm_0: tensor(39.5253, device='cuda:0') (tensor(26.4646, device='cuda:0')) + | > loss_gen: 2.110457420349121 (2.1165975794500227) + | > loss_kl: 2.564556121826172 (2.566230019744561) + | > loss_feat: 5.019462585449219 (4.653031446495836) + | > loss_mel: 20.75771141052246 (20.557811737060547) + | > loss_duration: 1.343770980834961 (1.320661727263003) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.795957565307617 (31.214332269162547) + | > grad_norm_1: tensor(1585.7992, device='cuda:0') (tensor(1037.1053, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.7983 (0.7891723087855748) + | > loader_time: 0.0093 (0.009525892685870735) + + + --> TIME: 2024-06-25 12:44:03 -- STEP: 74/591 -- GLOBAL_STEP: 36125 + | > loss_disc: 2.663087844848633 (2.6246765368693574) + | > loss_disc_real_0: 0.18267573416233063 (0.19817590753774386) + | > loss_disc_real_1: 0.2137371152639389 (0.22537352265538396) + | > loss_disc_real_2: 0.25435855984687805 (0.23125524255069527) + | > loss_disc_real_3: 0.23483070731163025 (0.23374102385462942) + | > loss_disc_real_4: 0.19040420651435852 (0.20208284812601837) + | > loss_disc_real_5: 0.24220459163188934 (0.23927441601817673) + | > loss_0: 2.663087844848633 (2.6246765368693574) + | > grad_norm_0: tensor(17.0002, device='cuda:0') (tensor(28.9956, device='cuda:0')) + | > loss_gen: 1.9736855030059814 (2.148379420911944) + | > loss_kl: 2.394829511642456 (2.5494662297738566) + | > loss_feat: 4.359593391418457 (4.707735248514125) + | > loss_mel: 20.894954681396484 (20.584741076907594) + | > loss_duration: 1.3551933765411377 (1.326133620094609) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.97825813293457 (31.316455531764674) + | > grad_norm_1: tensor(650.1733, device='cuda:0') (tensor(1013.7028, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.8094 (0.7983704386530697) + | > loader_time: 0.0119 (0.009821054097768423) + + + --> TIME: 2024-06-25 12:44:24 -- STEP: 99/591 -- GLOBAL_STEP: 36150 + | > loss_disc: 2.7725155353546143 (2.6213496815074566) + | > loss_disc_real_0: 0.26243412494659424 (0.19856329615971055) + | > loss_disc_real_1: 0.2505210041999817 (0.22509518160362435) + | > loss_disc_real_2: 0.26756590604782104 (0.23235271404487917) + | > loss_disc_real_3: 0.21649552881717682 (0.23415166937341594) + | > loss_disc_real_4: 0.20861466228961945 (0.2028229980908259) + | > loss_disc_real_5: 0.30464842915534973 (0.2397048608823256) + | > loss_0: 2.7725155353546143 (2.6213496815074566) + | > grad_norm_0: tensor(34.2365, device='cuda:0') (tensor(28.9530, device='cuda:0')) + | > loss_gen: 1.7298637628555298 (2.149446021426808) + | > loss_kl: 2.4658315181732178 (2.5405705745774076) + | > loss_feat: 4.288211822509766 (4.697434348289414) + | > loss_mel: 19.83251190185547 (20.62238332960341) + | > loss_duration: 1.3296879529953003 (1.3333504043444238) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.646106719970703 (31.343184557828035) + | > grad_norm_1: tensor(352.4065, device='cuda:0') (tensor(1096.3748, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.8269 (0.8057524676274773) + | > loader_time: 0.011 (0.010188863735006309) + + + --> TIME: 2024-06-25 12:44:45 -- STEP: 124/591 -- GLOBAL_STEP: 36175 + | > loss_disc: 2.642909288406372 (2.6163480589466706) + | > loss_disc_real_0: 0.18159955739974976 (0.19716078800059134) + | > loss_disc_real_1: 0.18938562273979187 (0.2241833459946417) + | > loss_disc_real_2: 0.21541829407215118 (0.2316124978805742) + | > loss_disc_real_3: 0.26071977615356445 (0.23390705547025126) + | > loss_disc_real_4: 0.20899073779582977 (0.20294394674560717) + | > loss_disc_real_5: 0.24556565284729004 (0.2390709709255926) + | > loss_0: 2.642909288406372 (2.6163480589466706) + | > grad_norm_0: tensor(19.2924, device='cuda:0') (tensor(28.6861, device='cuda:0')) + | > loss_gen: 1.9818449020385742 (2.1474010146433318) + | > loss_kl: 2.3331470489501953 (2.53157313023844) + | > loss_feat: 4.632222652435303 (4.701129740284335) + | > loss_mel: 20.35296058654785 (20.608449628276208) + | > loss_duration: 1.3515348434448242 (1.3341628591860495) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.651710510253906 (31.32271622073266) + | > grad_norm_1: tensor(490.8200, device='cuda:0') (tensor(1113.2202, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.8547 (0.8131291443301787) + | > loader_time: 0.0119 (0.010454916184948333) + + + --> TIME: 2024-06-25 12:45:07 -- STEP: 149/591 -- GLOBAL_STEP: 36200 + | > loss_disc: 2.6912481784820557 (2.613871971232779) + | > loss_disc_real_0: 0.19910353422164917 (0.19671846436174126) + | > loss_disc_real_1: 0.23004500567913055 (0.22367992957166377) + | > loss_disc_real_2: 0.2518163323402405 (0.2314528787896137) + | > loss_disc_real_3: 0.22802646458148956 (0.2333907848036529) + | > loss_disc_real_4: 0.15338698029518127 (0.2036814876150765) + | > loss_disc_real_5: 0.22675621509552002 (0.2384407408285461) + | > loss_0: 2.6912481784820557 (2.613871971232779) + | > grad_norm_0: tensor(17.4150, device='cuda:0') (tensor(28.4228, device='cuda:0')) + | > loss_gen: 1.9139949083328247 (2.141552621886235) + | > loss_kl: 2.53136944770813 (2.517912704672588) + | > loss_feat: 4.601144313812256 (4.686538782695797) + | > loss_mel: 20.431528091430664 (20.57247421725484) + | > loss_duration: 1.3462870121002197 (1.333129793205517) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.824323654174805 (31.251608035708433) + | > grad_norm_1: tensor(884.6751, device='cuda:0') (tensor(1114.7695, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.8632 (0.8187628624423239) + | > loader_time: 0.0126 (0.010674159798846148) + + + --> TIME: 2024-06-25 12:45:29 -- STEP: 174/591 -- GLOBAL_STEP: 36225 + | > loss_disc: 2.616518259048462 (2.6104464667967) + | > loss_disc_real_0: 0.18661731481552124 (0.19652150618447659) + | > loss_disc_real_1: 0.2873355448246002 (0.22360691779303823) + | > loss_disc_real_2: 0.22670447826385498 (0.2320666954606429) + | > loss_disc_real_3: 0.23543792963027954 (0.23317138970583334) + | > loss_disc_real_4: 0.24688979983329773 (0.20144813297980133) + | > loss_disc_real_5: 0.2364349216222763 (0.23799439639538184) + | > loss_0: 2.616518259048462 (2.6104464667967) + | > grad_norm_0: tensor(38.3707, device='cuda:0') (tensor(28.4795, device='cuda:0')) + | > loss_gen: 2.1885902881622314 (2.1427805512800995) + | > loss_kl: 2.5081119537353516 (2.510504525283287) + | > loss_feat: 4.410312652587891 (4.688173206373194) + | > loss_mel: 20.548303604125977 (20.562200557226408) + | > loss_duration: 1.3341851234436035 (1.3312862830600511) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.989503860473633 (31.234945034158642) + | > grad_norm_1: tensor(1145.0361, device='cuda:0') (tensor(1131.2588, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.8567 (0.8248056022600195) + | > loader_time: 0.0128 (0.010905708389720695) + + + --> TIME: 2024-06-25 12:45:51 -- STEP: 199/591 -- GLOBAL_STEP: 36250 + | > loss_disc: 2.5217764377593994 (2.6089205933575648) + | > loss_disc_real_0: 0.12725606560707092 (0.19580311193388314) + | > loss_disc_real_1: 0.19821040332317352 (0.22308966187975515) + | > loss_disc_real_2: 0.2137865573167801 (0.23182802381527484) + | > loss_disc_real_3: 0.20992374420166016 (0.23292773932068792) + | > loss_disc_real_4: 0.1641530841588974 (0.2025229744051569) + | > loss_disc_real_5: 0.22486358880996704 (0.23813940854228322) + | > loss_0: 2.5217764377593994 (2.6089205933575648) + | > grad_norm_0: tensor(25.2308, device='cuda:0') (tensor(28.0874, device='cuda:0')) + | > loss_gen: 2.3564882278442383 (2.137833679740753) + | > loss_kl: 2.5368900299072266 (2.5098476565662944) + | > loss_feat: 4.7989583015441895 (4.677294791044306) + | > loss_mel: 20.328433990478516 (20.558045871293707) + | > loss_duration: 1.3308581113815308 (1.3307645057314) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.35162925720215 (31.2137864271001) + | > grad_norm_1: tensor(1306.9431, device='cuda:0') (tensor(1131.4675, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.8573 (0.8310828089115009) + | > loader_time: 0.0122 (0.011081327745063816) + + + --> TIME: 2024-06-25 12:46:13 -- STEP: 224/591 -- GLOBAL_STEP: 36275 + | > loss_disc: 2.709725856781006 (2.607800558209419) + | > loss_disc_real_0: 0.21724095940589905 (0.1955738023389131) + | > loss_disc_real_1: 0.22658248245716095 (0.22297730423243983) + | > loss_disc_real_2: 0.2762775123119354 (0.2316320220540677) + | > loss_disc_real_3: 0.2231336086988449 (0.2328993693393256) + | > loss_disc_real_4: 0.1904289275407791 (0.20209729528453732) + | > loss_disc_real_5: 0.23686346411705017 (0.2379354935671602) + | > loss_0: 2.709725856781006 (2.607800558209419) + | > grad_norm_0: tensor(25.8464, device='cuda:0') (tensor(28.2168, device='cuda:0')) + | > loss_gen: 2.115485906600952 (2.1383749081620165) + | > loss_kl: 2.642052173614502 (2.5081560601081168) + | > loss_feat: 4.51823091506958 (4.676159096615659) + | > loss_mel: 20.684783935546875 (20.552474626473003) + | > loss_duration: 1.346412181854248 (1.3300901454474239) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.306964874267578 (31.205254767622268) + | > grad_norm_1: tensor(1334.9680, device='cuda:0') (tensor(1127.1898, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.8777 (0.836436753826482) + | > loader_time: 0.0158 (0.011403002909251617) + + + --> TIME: 2024-06-25 12:46:36 -- STEP: 249/591 -- GLOBAL_STEP: 36300 + | > loss_disc: 2.629411458969116 (2.608117660844182) + | > loss_disc_real_0: 0.2772490382194519 (0.19642231921594308) + | > loss_disc_real_1: 0.2209264487028122 (0.22284930615300635) + | > loss_disc_real_2: 0.24064117670059204 (0.2317852763287035) + | > loss_disc_real_3: 0.2471846491098404 (0.2328955544285985) + | > loss_disc_real_4: 0.21205903589725494 (0.20179528671215816) + | > loss_disc_real_5: 0.23255065083503723 (0.2380074390086783) + | > loss_0: 2.629411458969116 (2.608117660844182) + | > grad_norm_0: tensor(29.5810, device='cuda:0') (tensor(28.4937, device='cuda:0')) + | > loss_gen: 1.9594640731811523 (2.1369097879134036) + | > loss_kl: 2.4074559211730957 (2.504184230743163) + | > loss_feat: 4.452635765075684 (4.673781295378049) + | > loss_mel: 20.5991153717041 (20.546791704782983) + | > loss_duration: 1.3627227544784546 (1.3307321674852481) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.78139305114746 (31.192399120713812) + | > grad_norm_1: tensor(1073.5170, device='cuda:0') (tensor(1133.8508, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.9097 (0.8422446681792477) + | > loader_time: 0.0165 (0.011733717707745039) + + + --> TIME: 2024-06-25 12:46:59 -- STEP: 274/591 -- GLOBAL_STEP: 36325 + | > loss_disc: 2.57676100730896 (2.6083923543456695) + | > loss_disc_real_0: 0.15079936385154724 (0.19579239847668767) + | > loss_disc_real_1: 0.21046578884124756 (0.22264429714775438) + | > loss_disc_real_2: 0.21724793314933777 (0.23156300497098561) + | > loss_disc_real_3: 0.22226709127426147 (0.23273915074167462) + | > loss_disc_real_4: 0.1715668886899948 (0.20289606581965502) + | > loss_disc_real_5: 0.2142542004585266 (0.2379879448426901) + | > loss_0: 2.57676100730896 (2.6083923543456695) + | > grad_norm_0: tensor(27.2263, device='cuda:0') (tensor(28.0627, device='cuda:0')) + | > loss_gen: 2.1925389766693115 (2.1312048635343572) + | > loss_kl: 2.4575953483581543 (2.5058259337487883) + | > loss_feat: 4.852025985717773 (4.6587590457749215) + | > loss_mel: 21.16914176940918 (20.52826091320844) + | > loss_duration: 1.3588638305664062 (1.3306749454379947) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.03016662597656 (31.15472562469705) + | > grad_norm_1: tensor(1311.7303, device='cuda:0') (tensor(1131.4801, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.9018 (0.8482109560583629) + | > loader_time: 0.0161 (0.012027380240224574) + + + --> TIME: 2024-06-25 12:47:23 -- STEP: 299/591 -- GLOBAL_STEP: 36350 + | > loss_disc: 2.579732894897461 (2.6084704981201066) + | > loss_disc_real_0: 0.21812663972377777 (0.19607926371902926) + | > loss_disc_real_1: 0.25922101736068726 (0.22265643100674734) + | > loss_disc_real_2: 0.2174050211906433 (0.23125955273076443) + | > loss_disc_real_3: 0.2804696559906006 (0.23288038328140476) + | > loss_disc_real_4: 0.26882657408714294 (0.2031373093807977) + | > loss_disc_real_5: 0.23793891072273254 (0.2379846837010272) + | > loss_0: 2.579732894897461 (2.6084704981201066) + | > grad_norm_0: tensor(45.9834, device='cuda:0') (tensor(28.5363, device='cuda:0')) + | > loss_gen: 2.216792583465576 (2.132991269679373) + | > loss_kl: 2.3892176151275635 (2.5043168976952797) + | > loss_feat: 4.882420539855957 (4.658624157857739) + | > loss_mel: 19.80401039123535 (20.522369754752972) + | > loss_duration: 1.3587472438812256 (1.3314887795559933) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.65118980407715 (31.14979077980271) + | > grad_norm_1: tensor(1276.2322, device='cuda:0') (tensor(1139.8092, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.9119 (0.8542698681553871) + | > loader_time: 0.016 (0.012286025146178176) + + + --> TIME: 2024-06-25 12:47:46 -- STEP: 324/591 -- GLOBAL_STEP: 36375 + | > loss_disc: 2.558915138244629 (2.609897552449026) + | > loss_disc_real_0: 0.21946093440055847 (0.19611516811045607) + | > loss_disc_real_1: 0.21043755114078522 (0.22278641656409076) + | > loss_disc_real_2: 0.2250874936580658 (0.23142814249904067) + | > loss_disc_real_3: 0.20409928262233734 (0.23294237646975635) + | > loss_disc_real_4: 0.2193850874900818 (0.2031783212068273) + | > loss_disc_real_5: 0.22389504313468933 (0.23803902155271284) + | > loss_0: 2.558915138244629 (2.609897552449026) + | > grad_norm_0: tensor(43.0971, device='cuda:0') (tensor(28.6982, device='cuda:0')) + | > loss_gen: 2.2260148525238037 (2.131420380539366) + | > loss_kl: 2.5447046756744385 (2.5046403834849222) + | > loss_feat: 4.837745189666748 (4.654506471421985) + | > loss_mel: 20.730411529541016 (20.5172996226652) + | > loss_duration: 1.341813087463379 (1.3321904297228209) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.68069076538086 (31.14005721056903) + | > grad_norm_1: tensor(1567.3582, device='cuda:0') (tensor(1138.7538, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.9324 (0.8595070397412338) + | > loader_time: 0.0144 (0.012512927438005991) + + + --> TIME: 2024-06-25 12:48:10 -- STEP: 349/591 -- GLOBAL_STEP: 36400 + | > loss_disc: 2.6549344062805176 (2.610494086257368) + | > loss_disc_real_0: 0.23679855465888977 (0.19634363563320364) + | > loss_disc_real_1: 0.24900434911251068 (0.2229160584061057) + | > loss_disc_real_2: 0.2396906465291977 (0.2313500340910559) + | > loss_disc_real_3: 0.23029384016990662 (0.2329494127400625) + | > loss_disc_real_4: 0.23946309089660645 (0.20348506540401626) + | > loss_disc_real_5: 0.27276724576950073 (0.2377672580986105) + | > loss_0: 2.6549344062805176 (2.610494086257368) + | > grad_norm_0: tensor(20.7571, device='cuda:0') (tensor(28.5229, device='cuda:0')) + | > loss_gen: 2.0418100357055664 (2.1276083498083094) + | > loss_kl: 2.5692903995513916 (2.503768321094677) + | > loss_feat: 4.430975437164307 (4.643049477847737) + | > loss_mel: 20.44034194946289 (20.51704043850173) + | > loss_duration: 1.349466323852539 (1.3337169957365893) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.831884384155273 (31.125183498961878) + | > grad_norm_1: tensor(1136.2777, device='cuda:0') (tensor(1130.4294, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.9352 (0.8652667063355106) + | > loader_time: 0.0175 (0.012793040207258953) + + + --> TIME: 2024-06-25 12:48:35 -- STEP: 374/591 -- GLOBAL_STEP: 36425 + | > loss_disc: 2.6574599742889404 (2.613224821932177) + | > loss_disc_real_0: 0.17874011397361755 (0.19627824949628536) + | > loss_disc_real_1: 0.23046815395355225 (0.22274941544641147) + | > loss_disc_real_2: 0.21459104120731354 (0.2312770246423502) + | > loss_disc_real_3: 0.2483992576599121 (0.2329118646641466) + | > loss_disc_real_4: 0.22864994406700134 (0.20519772329352765) + | > loss_disc_real_5: 0.23233620822429657 (0.2378768239588661) + | > loss_0: 2.6574599742889404 (2.613224821932177) + | > grad_norm_0: tensor(20.7106, device='cuda:0') (tensor(27.9776, device='cuda:0')) + | > loss_gen: 2.005401134490967 (2.119706318977683) + | > loss_kl: 2.4698143005371094 (2.5019827513771262) + | > loss_feat: 4.159318447113037 (4.623155737305707) + | > loss_mel: 20.912174224853516 (20.515596445868987) + | > loss_duration: 1.3245185613632202 (1.335267000976093) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.871227264404297 (31.095708163664302) + | > grad_norm_1: tensor(866.4574, device='cuda:0') (tensor(1104.3737, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.9629 (0.870925822359993) + | > loader_time: 0.0178 (0.013051722776443566) + + + --> TIME: 2024-06-25 12:49:00 -- STEP: 399/591 -- GLOBAL_STEP: 36450 + | > loss_disc: 2.752490997314453 (2.616737603543696) + | > loss_disc_real_0: 0.15833735466003418 (0.19701278894990315) + | > loss_disc_real_1: 0.19819889962673187 (0.22282206937483975) + | > loss_disc_real_2: 0.22157227993011475 (0.23138294314830227) + | > loss_disc_real_3: 0.21042503416538239 (0.23321674962091565) + | > loss_disc_real_4: 0.18664945662021637 (0.20565003860639158) + | > loss_disc_real_5: 0.24984833598136902 (0.23774493999200358) + | > loss_0: 2.752490997314453 (2.616737603543696) + | > grad_norm_0: tensor(15.7655, device='cuda:0') (tensor(27.8567, device='cuda:0')) + | > loss_gen: 1.9729208946228027 (2.115826428384709) + | > loss_kl: 2.4594550132751465 (2.4997207012989184) + | > loss_feat: 4.235047340393066 (4.60914088968645) + | > loss_mel: 20.172643661499023 (20.51055124230253) + | > loss_duration: 1.3517512083053589 (1.3362832935830407) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.191818237304688 (31.071522466520918) + | > grad_norm_1: tensor(778.6840, device='cuda:0') (tensor(1102.6355, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 1.0198 (0.8786545684164331) + | > loader_time: 0.0189 (0.013317936345150598) + + + --> TIME: 2024-06-25 12:49:25 -- STEP: 424/591 -- GLOBAL_STEP: 36475 + | > loss_disc: 2.6011362075805664 (2.6160452185936705) + | > loss_disc_real_0: 0.18470653891563416 (0.19678604369104477) + | > loss_disc_real_1: 0.17939138412475586 (0.22269126080538867) + | > loss_disc_real_2: 0.20827142894268036 (0.23132612686252818) + | > loss_disc_real_3: 0.1745755523443222 (0.23319436131783253) + | > loss_disc_real_4: 0.18034592270851135 (0.20566471253152732) + | > loss_disc_real_5: 0.23652668297290802 (0.23787253227014588) + | > loss_0: 2.6011362075805664 (2.6160452185936705) + | > grad_norm_0: tensor(17.8902, device='cuda:0') (tensor(28.0535, device='cuda:0')) + | > loss_gen: 2.104513168334961 (2.115372425542687) + | > loss_kl: 2.414867401123047 (2.497774511575701) + | > loss_feat: 4.54470157623291 (4.60011718408117) + | > loss_mel: 19.856327056884766 (20.484508815801366) + | > loss_duration: 1.358883023262024 (1.3370902307753294) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.2792911529541 (31.03486308511698) + | > grad_norm_1: tensor(1080.7657, device='cuda:0') (tensor(1111.2111, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 1.035 (0.8853663093638872) + | > loader_time: 0.0184 (0.013611415647110851) + + + --> TIME: 2024-06-25 12:49:51 -- STEP: 449/591 -- GLOBAL_STEP: 36500 + | > loss_disc: 2.6880855560302734 (2.6170307405807405) + | > loss_disc_real_0: 0.3160170316696167 (0.19723814213129892) + | > loss_disc_real_1: 0.23564349114894867 (0.22294772995474077) + | > loss_disc_real_2: 0.22290600836277008 (0.2314060464800068) + | > loss_disc_real_3: 0.2503184676170349 (0.23326438233953276) + | > loss_disc_real_4: 0.19815443456172943 (0.2051484437953922) + | > loss_disc_real_5: 0.22293582558631897 (0.23805412089240577) + | > loss_0: 2.6880855560302734 (2.6170307405807405) + | > grad_norm_0: tensor(31.4135, device='cuda:0') (tensor(28.2683, device='cuda:0')) + | > loss_gen: 2.1320223808288574 (2.1167678036509203) + | > loss_kl: 2.5230350494384766 (2.4961449494606156) + | > loss_feat: 4.364905834197998 (4.59334558898994) + | > loss_mel: 20.3653621673584 (20.48430288183131) + | > loss_duration: 1.3511106967926025 (1.3382237244289543) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.73643684387207 (31.028784872959875) + | > grad_norm_1: tensor(1024.4178, device='cuda:0') (tensor(1108.0015, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 0.9979 (0.8923094442533227) + | > loader_time: 0.0172 (0.01388508121261087) + + + --> TIME: 2024-06-25 12:50:17 -- STEP: 474/591 -- GLOBAL_STEP: 36525 + | > loss_disc: 2.6724095344543457 (2.6178169537194176) + | > loss_disc_real_0: 0.3067011833190918 (0.19730148862119976) + | > loss_disc_real_1: 0.25059613585472107 (0.22303018291414037) + | > loss_disc_real_2: 0.2311447262763977 (0.2316567320298042) + | > loss_disc_real_3: 0.2665539085865021 (0.23333030498983487) + | > loss_disc_real_4: 0.1978963166475296 (0.20518381888919246) + | > loss_disc_real_5: 0.2613331973552704 (0.23821301145266882) + | > loss_0: 2.6724095344543457 (2.6178169537194176) + | > grad_norm_0: tensor(35.1176, device='cuda:0') (tensor(28.4655, device='cuda:0')) + | > loss_gen: 2.026210069656372 (2.115782487241526) + | > loss_kl: 2.604532480239868 (2.495182119341341) + | > loss_feat: 4.54372501373291 (4.582753903755158) + | > loss_mel: 20.262117385864258 (20.468469338075014) + | > loss_duration: 1.325085997581482 (1.3394502859075361) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.76167106628418 (31.00163806641655) + | > grad_norm_1: tensor(1381.5798, device='cuda:0') (tensor(1112.0070, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 1.0121 (0.8986333357130929) + | > loader_time: 0.0162 (0.014126489434061173) + + + --> TIME: 2024-06-25 12:50:44 -- STEP: 499/591 -- GLOBAL_STEP: 36550 + | > loss_disc: 2.6210551261901855 (2.6192247896251803) + | > loss_disc_real_0: 0.18748807907104492 (0.19737939305140648) + | > loss_disc_real_1: 0.24425838887691498 (0.2229754683787455) + | > loss_disc_real_2: 0.2315022349357605 (0.2315347323377051) + | > loss_disc_real_3: 0.22121025621891022 (0.23337316259234128) + | > loss_disc_real_4: 0.21702739596366882 (0.20576484574523868) + | > loss_disc_real_5: 0.23220233619213104 (0.23816418913538326) + | > loss_0: 2.6210551261901855 (2.6192247896251803) + | > grad_norm_0: tensor(24.3166, device='cuda:0') (tensor(28.2760, device='cuda:0')) + | > loss_gen: 2.0918471813201904 (2.112149145416839) + | > loss_kl: 2.4467995166778564 (2.495361650157311) + | > loss_feat: 4.483330249786377 (4.5686804096780005) + | > loss_mel: 20.042217254638672 (20.44922113131903) + | > loss_duration: 1.3672327995300293 (1.3408139038659288) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.431427001953125 (30.966226157301175) + | > grad_norm_1: tensor(1271.1110, device='cuda:0') (tensor(1103.8333, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 1.0485 (0.9057486998533201) + | > loader_time: 0.0168 (0.014394131356585243) + + + --> TIME: 2024-06-25 12:51:11 -- STEP: 524/591 -- GLOBAL_STEP: 36575 + | > loss_disc: 2.701998710632324 (2.61975410102888) + | > loss_disc_real_0: 0.22007212042808533 (0.19732344487525588) + | > loss_disc_real_1: 0.24638216197490692 (0.22312322810633492) + | > loss_disc_real_2: 0.21111051738262177 (0.23158330359417972) + | > loss_disc_real_3: 0.2461474984884262 (0.23331300501946275) + | > loss_disc_real_4: 0.24176143109798431 (0.20587941124068423) + | > loss_disc_real_5: 0.24879670143127441 (0.2381759728696055) + | > loss_0: 2.701998710632324 (2.61975410102888) + | > grad_norm_0: tensor(21.0422, device='cuda:0') (tensor(28.2179, device='cuda:0')) + | > loss_gen: 2.0385615825653076 (2.111427927745206) + | > loss_kl: 2.4778554439544678 (2.493641882452348) + | > loss_feat: 4.181648254394531 (4.559249197253746) + | > loss_mel: 20.544572830200195 (20.436750040709512) + | > loss_duration: 1.3865160942077637 (1.3426369917756729) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.629154205322266 (30.94370597373438) + | > grad_norm_1: tensor(974.0355, device='cuda:0') (tensor(1102.3762, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 1.094 (0.9137933486290561) + | > loader_time: 0.0251 (0.014667196128204581) + + + --> TIME: 2024-06-25 12:51:40 -- STEP: 549/591 -- GLOBAL_STEP: 36600 + | > loss_disc: 2.639169692993164 (2.6209858627701506) + | > loss_disc_real_0: 0.25065088272094727 (0.19768023489548653) + | > loss_disc_real_1: 0.2043629139661789 (0.22308664705149686) + | > loss_disc_real_2: 0.2385290265083313 (0.23159862514619184) + | > loss_disc_real_3: 0.24868687987327576 (0.23336289652058337) + | > loss_disc_real_4: 0.20891906321048737 (0.20611922772692856) + | > loss_disc_real_5: 0.24785907566547394 (0.2382904315852511) + | > loss_0: 2.639169692993164 (2.6209858627701506) + | > grad_norm_0: tensor(38.8943, device='cuda:0') (tensor(28.3802, device='cuda:0')) + | > loss_gen: 2.0902552604675293 (2.108891785253374) + | > loss_kl: 2.3489913940429688 (2.4907348025691984) + | > loss_feat: 4.12964391708374 (4.544791474368404) + | > loss_mel: 19.54915428161621 (20.41581744835024) + | > loss_duration: 1.3575671911239624 (1.3442604995598988) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.47561264038086 (30.90449594799938) + | > grad_norm_1: tensor(1283.7207, device='cuda:0') (tensor(1106.3622, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 1.1339 (0.9226607455581918) + | > loader_time: 0.0189 (0.0149688442765255) + + + --> TIME: 2024-06-25 12:52:09 -- STEP: 574/591 -- GLOBAL_STEP: 36625 + | > loss_disc: 2.568924903869629 (2.6208373058963734) + | > loss_disc_real_0: 0.22766317427158356 (0.1975379695122874) + | > loss_disc_real_1: 0.20333810150623322 (0.22304348302323654) + | > loss_disc_real_2: 0.2508709132671356 (0.2316828739508104) + | > loss_disc_real_3: 0.2660733759403229 (0.2334635904649409) + | > loss_disc_real_4: 0.19458046555519104 (0.20612107924727616) + | > loss_disc_real_5: 0.2461887151002884 (0.2381978945138147) + | > loss_0: 2.568924903869629 (2.6208373058963734) + | > grad_norm_0: tensor(30.1504, device='cuda:0') (tensor(28.3821, device='cuda:0')) + | > loss_gen: 2.1478474140167236 (2.107295793731036) + | > loss_kl: 2.4430463314056396 (2.4885600011938553) + | > loss_feat: 4.667983055114746 (4.534251291577412) + | > loss_mel: 20.399555206298828 (20.396796336157394) + | > loss_duration: 1.3829007148742676 (1.3459021231023274) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.04133415222168 (30.872805489064923) + | > grad_norm_1: tensor(1167.6428, device='cuda:0') (tensor(1105.3414, device='cuda:0')) + | > current_lr_0: 0.00019848070471685067 + | > current_lr_1: 0.00019848070471685067 + | > step_time: 1.1281 (0.9326178537428589) + | > loader_time: 0.0207 (0.015312673738193843) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003730606525502306 (+1.2281093191593168e-05) + | > avg_loss_disc: 2.7009383110289877 (+0.10196272363053982) + | > avg_loss_disc_real_0: 0.2397217370094137 (+0.05746379478814756) + | > avg_loss_disc_real_1: 0.19637427051016626 (-0.027698433779655612) + | > avg_loss_disc_real_2: 0.20733564775040808 (-0.04662754973198513) + | > avg_loss_disc_real_3: 0.18923871853250138 (-0.14560128431370908) + | > avg_loss_disc_real_4: 0.2490743100643158 (+0.03813936894244335) + | > avg_loss_disc_real_5: 0.22633922163476336 (-0.012701917201914686) + | > avg_loss_0: 2.7009383110289877 (+0.10196272363053982) + | > avg_loss_gen: 1.8551001371221338 (-0.34676984269568223) + | > avg_loss_kl: 2.577355090607988 (+0.04166045594722734) + | > avg_loss_feat: 4.149743191739346 (-0.3030734011467464) + | > avg_loss_mel: 20.8250658157024 (+0.6020538654733159) + | > avg_loss_duration: 1.36321955031537 (-0.0008995634444217693) + | > avg_loss_1: 30.77048382860549 (-0.007028539130029543) + + + > EPOCH: 63/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 12:52:37)  + + --> TIME: 2024-06-25 12:52:45 -- STEP: 8/591 -- GLOBAL_STEP: 36650 + | > loss_disc: 2.6829566955566406 (2.667823702096939) + | > loss_disc_real_0: 0.1616346389055252 (0.19598010927438736) + | > loss_disc_real_1: 0.1882432997226715 (0.228788360953331) + | > loss_disc_real_2: 0.24775120615959167 (0.23864861018955708) + | > loss_disc_real_3: 0.2370436191558838 (0.23973306268453598) + | > loss_disc_real_4: 0.2597728669643402 (0.23173044621944427) + | > loss_disc_real_5: 0.23862290382385254 (0.2346473801881075) + | > loss_0: 2.6829566955566406 (2.667823702096939) + | > grad_norm_0: tensor(14.5772, device='cuda:0') (tensor(13.7677, device='cuda:0')) + | > loss_gen: 2.03059720993042 (2.001712754368782) + | > loss_kl: 2.7373411655426025 (2.6397972404956818) + | > loss_feat: 4.521627902984619 (4.497918367385864) + | > loss_mel: 20.340831756591797 (20.629123210906982) + | > loss_duration: 1.3058452606201172 (1.3176974654197693) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.936243057250977 (31.086249113082886) + | > grad_norm_1: tensor(552.9615, device='cuda:0') (tensor(485.0762, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.774 (0.7720110714435577) + | > loader_time: 0.0085 (0.008625179529190063) + + + --> TIME: 2024-06-25 12:53:05 -- STEP: 33/591 -- GLOBAL_STEP: 36675 + | > loss_disc: 2.6973023414611816 (2.670963518547289) + | > loss_disc_real_0: 0.1930255889892578 (0.1988738246939399) + | > loss_disc_real_1: 0.22809302806854248 (0.22424279379122186) + | > loss_disc_real_2: 0.26666751503944397 (0.23162079128352078) + | > loss_disc_real_3: 0.2218436896800995 (0.2341230335560712) + | > loss_disc_real_4: 0.2710639238357544 (0.2356791663350481) + | > loss_disc_real_5: 0.24907055497169495 (0.23953442817384546) + | > loss_0: 2.6973023414611816 (2.670963518547289) + | > grad_norm_0: tensor(7.3210, device='cuda:0') (tensor(11.1832, device='cuda:0')) + | > loss_gen: 2.000214099884033 (2.0055363792361636) + | > loss_kl: 2.5850486755371094 (2.5931906049901787) + | > loss_feat: 4.369649410247803 (4.475112264806574) + | > loss_mel: 20.193904876708984 (20.591346393931993) + | > loss_duration: 1.328499674797058 (1.3097324551958025) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.47731590270996 (30.974917845292524) + | > grad_norm_1: tensor(180.8960, device='cuda:0') (tensor(328.9963, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.7981 (0.7846814574617328) + | > loader_time: 0.0095 (0.009470628969597092) + + + --> TIME: 2024-06-25 12:53:26 -- STEP: 58/591 -- GLOBAL_STEP: 36700 + | > loss_disc: 2.727090835571289 (2.655571855347732) + | > loss_disc_real_0: 0.11598468571901321 (0.19956822860343704) + | > loss_disc_real_1: 0.20669206976890564 (0.22258069957124776) + | > loss_disc_real_2: 0.22464695572853088 (0.23072444236484066) + | > loss_disc_real_3: 0.22658614814281464 (0.23280602822016025) + | > loss_disc_real_4: 0.2274206578731537 (0.2289661947509338) + | > loss_disc_real_5: 0.22165194153785706 (0.23651036697215047) + | > loss_0: 2.727090835571289 (2.655571855347732) + | > grad_norm_0: tensor(47.9481, device='cuda:0') (tensor(16.5422, device='cuda:0')) + | > loss_gen: 2.010502815246582 (2.033855378627778) + | > loss_kl: 2.5211431980133057 (2.5561023005123804) + | > loss_feat: 4.331373691558838 (4.508126357506061) + | > loss_mel: 20.29694366455078 (20.551515151714458) + | > loss_duration: 1.315990924835205 (1.319679173929938) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.4759521484375 (30.969278171144683) + | > grad_norm_1: tensor(571.6268, device='cuda:0') (tensor(535.4218, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.8336 (0.7946829549197493) + | > loader_time: 0.0098 (0.00972097906573065) + + + --> TIME: 2024-06-25 12:53:47 -- STEP: 83/591 -- GLOBAL_STEP: 36725 + | > loss_disc: 2.725982666015625 (2.6396118244492857) + | > loss_disc_real_0: 0.16116784512996674 (0.19958306211664015) + | > loss_disc_real_1: 0.24107113480567932 (0.22522597176482878) + | > loss_disc_real_2: 0.24297769367694855 (0.23025328136352172) + | > loss_disc_real_3: 0.23961803317070007 (0.23279838784631476) + | > loss_disc_real_4: 0.19010315835475922 (0.2182229780648128) + | > loss_disc_real_5: 0.2920023500919342 (0.23771538361009345) + | > loss_0: 2.725982666015625 (2.6396118244492857) + | > grad_norm_0: tensor(22.1980, device='cuda:0') (tensor(21.2821, device='cuda:0')) + | > loss_gen: 1.9272880554199219 (2.0882164010082387) + | > loss_kl: 2.4907920360565186 (2.5294369674590698) + | > loss_feat: 4.493319511413574 (4.616655964449227) + | > loss_mel: 20.457637786865234 (20.593671798706048) + | > loss_duration: 1.3460115194320679 (1.3269408122602713) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.715049743652344 (31.154921864888752) + | > grad_norm_1: tensor(961.2141, device='cuda:0') (tensor(680.3535, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.839 (0.8029669537601705) + | > loader_time: 0.0106 (0.009998111839754036) + + + --> TIME: 2024-06-25 12:54:08 -- STEP: 108/591 -- GLOBAL_STEP: 36750 + | > loss_disc: 2.5637519359588623 (2.637426431532259) + | > loss_disc_real_0: 0.19950813055038452 (0.1989251620101708) + | > loss_disc_real_1: 0.21397344768047333 (0.22404553410079744) + | > loss_disc_real_2: 0.21533280611038208 (0.23062117301203586) + | > loss_disc_real_3: 0.2737717032432556 (0.23335345379180378) + | > loss_disc_real_4: 0.17431572079658508 (0.21781331976806675) + | > loss_disc_real_5: 0.23055751621723175 (0.23805090164144835) + | > loss_0: 2.5637519359588623 (2.637426431532259) + | > grad_norm_0: tensor(26.0591, device='cuda:0') (tensor(21.8874, device='cuda:0')) + | > loss_gen: 2.1732728481292725 (2.0839331514305535) + | > loss_kl: 2.4492509365081787 (2.520545107346994) + | > loss_feat: 4.761078357696533 (4.595351338386535) + | > loss_mel: 19.88944435119629 (20.550680919929786) + | > loss_duration: 1.3612048625946045 (1.3315931801442749) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.63425064086914 (31.08210364094487) + | > grad_norm_1: tensor(1268.6426, device='cuda:0') (tensor(803.1393, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.8411 (0.8106794401451396) + | > loader_time: 0.0123 (0.010283635722266302) + + + --> TIME: 2024-06-25 12:54:30 -- STEP: 133/591 -- GLOBAL_STEP: 36775 + | > loss_disc: 2.6622605323791504 (2.6347692980802147) + | > loss_disc_real_0: 0.18746960163116455 (0.19921717994419255) + | > loss_disc_real_1: 0.27268362045288086 (0.22381973647533504) + | > loss_disc_real_2: 0.2517874538898468 (0.23088352214124866) + | > loss_disc_real_3: 0.24883241951465607 (0.23323312065655127) + | > loss_disc_real_4: 0.18804582953453064 (0.2154817226013743) + | > loss_disc_real_5: 0.23744145035743713 (0.2379642655526785) + | > loss_0: 2.6622605323791504 (2.6347692980802147) + | > grad_norm_0: tensor(32.6443, device='cuda:0') (tensor(23.7808, device='cuda:0')) + | > loss_gen: 1.970158338546753 (2.091659693789661) + | > loss_kl: 2.502242088317871 (2.51351517304442) + | > loss_feat: 4.438739776611328 (4.606914627820925) + | > loss_mel: 20.585012435913086 (20.52570601154987) + | > loss_duration: 1.3228681087493896 (1.3333086671685817) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.819019317626953 (31.071104150069388) + | > grad_norm_1: tensor(1688.0615, device='cuda:0') (tensor(898.0529, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.8286 (0.8180167997690074) + | > loader_time: 0.0113 (0.010512732025375936) + + + --> TIME: 2024-06-25 12:54:52 -- STEP: 158/591 -- GLOBAL_STEP: 36800 + | > loss_disc: 2.570314645767212 (2.6305112642577932) + | > loss_disc_real_0: 0.14783596992492676 (0.19836526802634896) + | > loss_disc_real_1: 0.24495813250541687 (0.2234405537572088) + | > loss_disc_real_2: 0.23195157945156097 (0.23075269898281822) + | > loss_disc_real_3: 0.253440797328949 (0.23295791909287247) + | > loss_disc_real_4: 0.20274575054645538 (0.21457234411677228) + | > loss_disc_real_5: 0.23821638524532318 (0.2377519262365148) + | > loss_0: 2.570314645767212 (2.6305112642577932) + | > grad_norm_0: tensor(20.0039, device='cuda:0') (tensor(24.6406, device='cuda:0')) + | > loss_gen: 2.0991594791412354 (2.0946537890011756) + | > loss_kl: 2.5041074752807617 (2.509151679051073) + | > loss_feat: 4.802253246307373 (4.609387307227411) + | > loss_mel: 20.469900131225586 (20.488415754294095) + | > loss_duration: 1.319165587425232 (1.3317770535432842) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.1945858001709 (31.03338551823097) + | > grad_norm_1: tensor(1326.4242, device='cuda:0') (tensor(949.0016, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.8407 (0.8256006980244117) + | > loader_time: 0.0113 (0.01080749306497694) + + + --> TIME: 2024-06-25 12:55:14 -- STEP: 183/591 -- GLOBAL_STEP: 36825 + | > loss_disc: 2.649111270904541 (2.6272877695781935) + | > loss_disc_real_0: 0.13113418221473694 (0.19798938241975558) + | > loss_disc_real_1: 0.2547759711742401 (0.22338579049527318) + | > loss_disc_real_2: 0.23152759671211243 (0.2305021268096778) + | > loss_disc_real_3: 0.1993744671344757 (0.23264592096128098) + | > loss_disc_real_4: 0.2503698766231537 (0.21289505365767766) + | > loss_disc_real_5: 0.2378704696893692 (0.23788670552232877) + | > loss_0: 2.649111270904541 (2.6272877695781935) + | > grad_norm_0: tensor(25.9382, device='cuda:0') (tensor(24.7099, device='cuda:0')) + | > loss_gen: 2.139110803604126 (2.0996800556860333) + | > loss_kl: 2.4169631004333496 (2.5106384311217433) + | > loss_feat: 4.865440845489502 (4.623455428034881) + | > loss_mel: 21.28915786743164 (20.483046547311258) + | > loss_duration: 1.3413156270980835 (1.330665783152554) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.05198669433594 (31.04748618537611) + | > grad_norm_1: tensor(1232.8376, device='cuda:0') (tensor(964.4622, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.9002 (0.8317882780168877) + | > loader_time: 0.0137 (0.011004959950681587) + + + --> TIME: 2024-06-25 12:55:37 -- STEP: 208/591 -- GLOBAL_STEP: 36850 + | > loss_disc: 2.648629665374756 (2.6296049941044584) + | > loss_disc_real_0: 0.15188392996788025 (0.19853533220549047) + | > loss_disc_real_1: 0.25568950176239014 (0.22362341201649263) + | > loss_disc_real_2: 0.2582963705062866 (0.23073680691707593) + | > loss_disc_real_3: 0.23498155176639557 (0.23261459909666043) + | > loss_disc_real_4: 0.29795846343040466 (0.2135259318523682) + | > loss_disc_real_5: 0.2330477386713028 (0.23805830613351786) + | > loss_0: 2.648629665374756 (2.6296049941044584) + | > grad_norm_0: tensor(42.6249, device='cuda:0') (tensor(25.1285, device='cuda:0')) + | > loss_gen: 2.1110589504241943 (2.0981185115300685) + | > loss_kl: 2.411384105682373 (2.5111228789274502) + | > loss_feat: 4.672677040100098 (4.61458025070337) + | > loss_mel: 20.464420318603516 (20.47397082585555) + | > loss_duration: 1.345327377319336 (1.3304281842250085) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.004867553710938 (31.028220616854153) + | > grad_norm_1: tensor(1623.9434, device='cuda:0') (tensor(991.1642, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.9174 (0.8387791571708826) + | > loader_time: 0.0144 (0.011241926596714896) + + + --> TIME: 2024-06-25 12:55:59 -- STEP: 233/591 -- GLOBAL_STEP: 36875 + | > loss_disc: 2.703218698501587 (2.6270362297352796) + | > loss_disc_real_0: 0.21562665700912476 (0.19804736690459848) + | > loss_disc_real_1: 0.2240191102027893 (0.22320464464472087) + | > loss_disc_real_2: 0.25174766778945923 (0.23084977857544697) + | > loss_disc_real_3: 0.20996540784835815 (0.232702389999009) + | > loss_disc_real_4: 0.23121356964111328 (0.21258496361996482) + | > loss_disc_real_5: 0.27648401260375977 (0.23799005579283308) + | > loss_0: 2.703218698501587 (2.6270362297352796) + | > grad_norm_0: tensor(5.7215, device='cuda:0') (tensor(25.6081, device='cuda:0')) + | > loss_gen: 1.8552109003067017 (2.0983111208600542) + | > loss_kl: 2.4010534286499023 (2.5057361729666905) + | > loss_feat: 4.363171577453613 (4.6153019323881095) + | > loss_mel: 19.896459579467773 (20.453708345797953) + | > loss_duration: 1.3298723697662354 (1.3300284894239232) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.845767974853516 (31.00308603278557) + | > grad_norm_1: tensor(151.4699, device='cuda:0') (tensor(1014.2414, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.8686 (0.8432343896366495) + | > loader_time: 0.0166 (0.011550143041324202) + + + --> TIME: 2024-06-25 12:56:22 -- STEP: 258/591 -- GLOBAL_STEP: 36900 + | > loss_disc: 2.548034191131592 (2.62927074118178) + | > loss_disc_real_0: 0.18988631665706635 (0.19796754887630777) + | > loss_disc_real_1: 0.20655487477779388 (0.22308971137963524) + | > loss_disc_real_2: 0.2279520332813263 (0.23104387111673058) + | > loss_disc_real_3: 0.2201177030801773 (0.23282743106747783) + | > loss_disc_real_4: 0.21352961659431458 (0.21409359054509983) + | > loss_disc_real_5: 0.223800927400589 (0.23791067469720692) + | > loss_0: 2.548034191131592 (2.62927074118178) + | > grad_norm_0: tensor(25.7043, device='cuda:0') (tensor(25.0505, device='cuda:0')) + | > loss_gen: 1.9342776536941528 (2.091865652753401) + | > loss_kl: 2.3875834941864014 (2.5066683357076123) + | > loss_feat: 4.728193283081055 (4.597918584365253) + | > loss_mel: 20.391128540039062 (20.45439809606981) + | > loss_duration: 1.3315304517745972 (1.3300061429193772) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.77271270751953 (30.980856762375943) + | > grad_norm_1: tensor(1359.2451, device='cuda:0') (tensor(997.1698, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.8979 (0.8491142600081688) + | > loader_time: 0.0137 (0.011817203935726665) + + + --> TIME: 2024-06-25 12:56:45 -- STEP: 283/591 -- GLOBAL_STEP: 36925 + | > loss_disc: 2.5406336784362793 (2.629334432076228) + | > loss_disc_real_0: 0.24247875809669495 (0.1979314905752563) + | > loss_disc_real_1: 0.20910289883613586 (0.22291857704467571) + | > loss_disc_real_2: 0.21066565811634064 (0.23063452199575338) + | > loss_disc_real_3: 0.22718170285224915 (0.232627055091066) + | > loss_disc_real_4: 0.18992677330970764 (0.21438129535833433) + | > loss_disc_real_5: 0.2262757271528244 (0.2379659500117858) + | > loss_0: 2.5406336784362793 (2.629334432076228) + | > grad_norm_0: tensor(33.4218, device='cuda:0') (tensor(25.1127, device='cuda:0')) + | > loss_gen: 2.0626449584960938 (2.0888134110521515) + | > loss_kl: 2.4920718669891357 (2.5049010940659593) + | > loss_feat: 4.960993766784668 (4.586557720238246) + | > loss_mel: 20.363605499267578 (20.44488741062555) + | > loss_duration: 1.3393725156784058 (1.3308829153384418) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.21868896484375 (30.956042508775692) + | > grad_norm_1: tensor(1269.0859, device='cuda:0') (tensor(1000.1063, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.8967 (0.853876156857494) + | > loader_time: 0.0132 (0.012090516174639911) + + + --> TIME: 2024-06-25 12:57:09 -- STEP: 308/591 -- GLOBAL_STEP: 36950 + | > loss_disc: 2.524296283721924 (2.627016461514808) + | > loss_disc_real_0: 0.19446581602096558 (0.1976609570397572) + | > loss_disc_real_1: 0.21972790360450745 (0.2226845295011223) + | > loss_disc_real_2: 0.22662387788295746 (0.23060870678587395) + | > loss_disc_real_3: 0.23442305624485016 (0.2327298651654999) + | > loss_disc_real_4: 0.2065383940935135 (0.21347652883692222) + | > loss_disc_real_5: 0.2572562098503113 (0.23793262141672047) + | > loss_0: 2.524296283721924 (2.627016461514808) + | > grad_norm_0: tensor(28.4725, device='cuda:0') (tensor(25.6237, device='cuda:0')) + | > loss_gen: 2.2166571617126465 (2.0921815022245642) + | > loss_kl: 2.509270191192627 (2.5045582474052126) + | > loss_feat: 4.848757266998291 (4.591903191108208) + | > loss_mel: 21.378625869750977 (20.447650915616524) + | > loss_duration: 1.3068604469299316 (1.3315510494368423) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.260169982910156 (30.967844870183374) + | > grad_norm_1: tensor(1451.6239, device='cuda:0') (tensor(1023.1074, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.923 (0.8594593243165449) + | > loader_time: 0.014 (0.012362374113751693) + + + --> TIME: 2024-06-25 12:57:33 -- STEP: 333/591 -- GLOBAL_STEP: 36975 + | > loss_disc: 2.641857624053955 (2.6254796795658892) + | > loss_disc_real_0: 0.19736814498901367 (0.1974020523575691) + | > loss_disc_real_1: 0.20941242575645447 (0.2226637063083706) + | > loss_disc_real_2: 0.21704910695552826 (0.23057543770508007) + | > loss_disc_real_3: 0.17904072999954224 (0.23261299313188674) + | > loss_disc_real_4: 0.21589119732379913 (0.21305441511822892) + | > loss_disc_real_5: 0.21839283406734467 (0.2379632145166397) + | > loss_0: 2.641857624053955 (2.6254796795658892) + | > grad_norm_0: tensor(48.6058, device='cuda:0') (tensor(25.9840, device='cuda:0')) + | > loss_gen: 2.180850028991699 (2.0952071169833184) + | > loss_kl: 2.4196739196777344 (2.5015504617948787) + | > loss_feat: 4.70414400100708 (4.596706698248697) + | > loss_mel: 20.112947463989258 (20.44099725832094) + | > loss_duration: 1.3511784076690674 (1.332470363682813) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.7687931060791 (30.966931849986583) + | > grad_norm_1: tensor(1878.5414, device='cuda:0') (tensor(1032.6449, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.9253 (0.8646543305199426) + | > loader_time: 0.0168 (0.012650270719785943) + + + --> TIME: 2024-06-25 12:57:57 -- STEP: 358/591 -- GLOBAL_STEP: 37000 + | > loss_disc: 2.5834531784057617 (2.627220373579912) + | > loss_disc_real_0: 0.21692639589309692 (0.19759126369680105) + | > loss_disc_real_1: 0.2232145071029663 (0.22302049539941649) + | > loss_disc_real_2: 0.2194928526878357 (0.23063606156983188) + | > loss_disc_real_3: 0.20383352041244507 (0.2328460771361543) + | > loss_disc_real_4: 0.2449856847524643 (0.2131970074839432) + | > loss_disc_real_5: 0.24772509932518005 (0.23799489118033948) + | > loss_0: 2.5834531784057617 (2.627220373579912) + | > grad_norm_0: tensor(25.1056, device='cuda:0') (tensor(26.0191, device='cuda:0')) + | > loss_gen: 2.1059837341308594 (2.09370998064233) + | > loss_kl: 2.443490505218506 (2.5005198490686253) + | > loss_feat: 4.348005294799805 (4.585762035247337) + | > loss_mel: 20.465417861938477 (20.43515096163617) + | > loss_duration: 1.3837391138076782 (1.3338167014734705) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.74663734436035 (30.948959505091832) + | > grad_norm_1: tensor(998.3466, device='cuda:0') (tensor(1031.5468, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.9213 (0.8705271469148178) + | > loader_time: 0.0182 (0.012915924274721621) + + + --> TIME: 2024-06-25 12:58:21 -- STEP: 383/591 -- GLOBAL_STEP: 37025 + | > loss_disc: 2.5529963970184326 (2.626052609311697) + | > loss_disc_real_0: 0.19416402280330658 (0.19730048111151155) + | > loss_disc_real_1: 0.214821919798851 (0.22297038752634282) + | > loss_disc_real_2: 0.20953096449375153 (0.23067159308777155) + | > loss_disc_real_3: 0.24688342213630676 (0.23276117156130838) + | > loss_disc_real_4: 0.20998218655586243 (0.2122694965130978) + | > loss_disc_real_5: 0.2083459049463272 (0.23781199578205536) + | > loss_0: 2.5529963970184326 (2.626052609311697) + | > grad_norm_0: tensor(33.7340, device='cuda:0') (tensor(26.1938, device='cuda:0')) + | > loss_gen: 2.4315171241760254 (2.094937350357793) + | > loss_kl: 2.476957321166992 (2.4987593967049295) + | > loss_feat: 5.1397175788879395 (4.583022673509766) + | > loss_mel: 20.558223724365234 (20.418781489678533) + | > loss_duration: 1.3456432819366455 (1.3351806094067529) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.952058792114258 (30.93068150751896) + | > grad_norm_1: tensor(1395.5228, device='cuda:0') (tensor(1036.1791, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.9615 (0.8761685149787922) + | > loader_time: 0.0181 (0.013210374443711558) + + + --> TIME: 2024-06-25 12:58:46 -- STEP: 408/591 -- GLOBAL_STEP: 37050 + | > loss_disc: 2.610502243041992 (2.6261715877289897) + | > loss_disc_real_0: 0.16302518546581268 (0.19750115213294825) + | > loss_disc_real_1: 0.1956799477338791 (0.22293636513253054) + | > loss_disc_real_2: 0.23870034515857697 (0.23086058293633602) + | > loss_disc_real_3: 0.2310754805803299 (0.2328351893480502) + | > loss_disc_real_4: 0.20706701278686523 (0.21179385724313118) + | > loss_disc_real_5: 0.24700313806533813 (0.2379382801990883) + | > loss_0: 2.610502243041992 (2.6261715877289897) + | > grad_norm_0: tensor(33.4745, device='cuda:0') (tensor(26.3868, device='cuda:0')) + | > loss_gen: 2.0727076530456543 (2.0943062375573556) + | > loss_kl: 2.4150025844573975 (2.4973996439400836) + | > loss_feat: 4.413924694061279 (4.576842236168247) + | > loss_mel: 20.125185012817383 (20.413391454547067) + | > loss_duration: 1.3535823822021484 (1.3364665937774325) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.380403518676758 (30.91840616394492) + | > grad_norm_1: tensor(1430.9806, device='cuda:0') (tensor(1041.8865, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 1.0099 (0.8825929252540364) + | > loader_time: 0.0195 (0.013506174087524409) + + + --> TIME: 2024-06-25 12:59:12 -- STEP: 433/591 -- GLOBAL_STEP: 37075 + | > loss_disc: 2.6024341583251953 (2.626489165748793) + | > loss_disc_real_0: 0.23551249504089355 (0.19732009373415976) + | > loss_disc_real_1: 0.20494046807289124 (0.22289000960888544) + | > loss_disc_real_2: 0.21644161641597748 (0.23077829581347678) + | > loss_disc_real_3: 0.22710919380187988 (0.23297178573200938) + | > loss_disc_real_4: 0.21440467238426208 (0.21229053971811476) + | > loss_disc_real_5: 0.24732020497322083 (0.23793841640338062) + | > loss_0: 2.6024341583251953 (2.626489165748793) + | > grad_norm_0: tensor(22.8014, device='cuda:0') (tensor(26.5158, device='cuda:0')) + | > loss_gen: 2.080566644668579 (2.092934827055722) + | > loss_kl: 2.423245906829834 (2.4964166023439405) + | > loss_feat: 4.448275566101074 (4.567210503593605) + | > loss_mel: 20.18916893005371 (20.394629383747912) + | > loss_duration: 1.3651998043060303 (1.3374128886788879) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.50645637512207 (30.888604201565713) + | > grad_norm_1: tensor(1164.2262, device='cuda:0') (tensor(1042.5496, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 1.0014 (0.8892479922975184) + | > loader_time: 0.0201 (0.013787126431013636) + + + --> TIME: 2024-06-25 12:59:38 -- STEP: 458/591 -- GLOBAL_STEP: 37100 + | > loss_disc: 2.4797091484069824 (2.6249476967940715) + | > loss_disc_real_0: 0.18239161372184753 (0.19721937140523085) + | > loss_disc_real_1: 0.22222818434238434 (0.22286603590026172) + | > loss_disc_real_2: 0.19415512681007385 (0.23076290520127685) + | > loss_disc_real_3: 0.2784278690814972 (0.2329150404146665) + | > loss_disc_real_4: 0.19966818392276764 (0.21171095462995845) + | > loss_disc_real_5: 0.2667580842971802 (0.23798052276846623) + | > loss_0: 2.4797091484069824 (2.6249476967940715) + | > grad_norm_0: tensor(21.7955, device='cuda:0') (tensor(26.6082, device='cuda:0')) + | > loss_gen: 2.378096342086792 (2.0948765251834316) + | > loss_kl: 2.5373826026916504 (2.494137943571832) + | > loss_feat: 4.831707000732422 (4.564549464846284) + | > loss_mel: 20.132213592529297 (20.3822389544358) + | > loss_duration: 1.375953197479248 (1.3383074059236522) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.255352020263672 (30.87411027600151) + | > grad_norm_1: tensor(2006.0460, device='cuda:0') (tensor(1050.9374, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 0.988 (0.8961169896688003) + | > loader_time: 0.0166 (0.014078330785426507) + + + --> TIME: 2024-06-25 13:00:04 -- STEP: 483/591 -- GLOBAL_STEP: 37125 + | > loss_disc: 2.628093719482422 (2.6249174148646475) + | > loss_disc_real_0: 0.1203247532248497 (0.19702642986369676) + | > loss_disc_real_1: 0.2281278818845749 (0.22290583229459837) + | > loss_disc_real_2: 0.22941066324710846 (0.230818776826434) + | > loss_disc_real_3: 0.24801106750965118 (0.23287538906432087) + | > loss_disc_real_4: 0.19776712357997894 (0.21139090546786662) + | > loss_disc_real_5: 0.2453916370868683 (0.23796031496046) + | > loss_0: 2.628093719482422 (2.6249174148646475) + | > grad_norm_0: tensor(31.7292, device='cuda:0') (tensor(26.9683, device='cuda:0')) + | > loss_gen: 2.1259732246398926 (2.0946836570281646) + | > loss_kl: 2.4303276538848877 (2.4926262578115206) + | > loss_feat: 4.487708568572998 (4.558022692583611) + | > loss_mel: 20.178680419921875 (20.368916537203905) + | > loss_duration: 1.3890419006347656 (1.3397413001050615) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.611732482910156 (30.853990416595902) + | > grad_norm_1: tensor(1357.6014, device='cuda:0') (tensor(1060.1182, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 1.0154 (0.9030877729380353) + | > loader_time: 0.0208 (0.014369240952327875) + + + --> TIME: 2024-06-25 13:00:33 -- STEP: 508/591 -- GLOBAL_STEP: 37150 + | > loss_disc: 2.5849931240081787 (2.625023144436636) + | > loss_disc_real_0: 0.2059451937675476 (0.19717480421183617) + | > loss_disc_real_1: 0.23005026578903198 (0.22289731060191403) + | > loss_disc_real_2: 0.25113144516944885 (0.23088132012195475) + | > loss_disc_real_3: 0.19536705315113068 (0.23298053971425756) + | > loss_disc_real_4: 0.19951984286308289 (0.21128401347971337) + | > loss_disc_real_5: 0.24218076467514038 (0.23787799941038523) + | > loss_0: 2.5849931240081787 (2.625023144436636) + | > grad_norm_0: tensor(30.8289, device='cuda:0') (tensor(26.9500, device='cuda:0')) + | > loss_gen: 2.2077910900115967 (2.094176996880627) + | > loss_kl: 2.549475908279419 (2.491832468453355) + | > loss_feat: 4.8285603523254395 (4.551196833764476) + | > loss_mel: 19.958127975463867 (20.35649628526583) + | > loss_duration: 1.3431060314178467 (1.3413540943870397) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.887060165405273 (30.835056650356982) + | > grad_norm_1: tensor(1277.6923, device='cuda:0') (tensor(1061.6951, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 1.1779 (0.9136896222595154) + | > loader_time: 0.021 (0.01469294339653075) + + + --> TIME: 2024-06-25 13:01:03 -- STEP: 533/591 -- GLOBAL_STEP: 37175 + | > loss_disc: 2.6177825927734375 (2.6258487741674337) + | > loss_disc_real_0: 0.1771627962589264 (0.1971719687696246) + | > loss_disc_real_1: 0.20704865455627441 (0.22313363260220856) + | > loss_disc_real_2: 0.23609544336795807 (0.23096975463304467) + | > loss_disc_real_3: 0.22072918713092804 (0.23313814460336604) + | > loss_disc_real_4: 0.20328815281391144 (0.211110485651256) + | > loss_disc_real_5: 0.23781010508537292 (0.23788538969368242) + | > loss_0: 2.6177825927734375 (2.6258487741674337) + | > grad_norm_0: tensor(26.0919, device='cuda:0') (tensor(27.1437, device='cuda:0')) + | > loss_gen: 2.075653314590454 (2.0953674683204047) + | > loss_kl: 2.4238502979278564 (2.4895008831489376) + | > loss_feat: 4.196612358093262 (4.546468221671587) + | > loss_mel: 19.961076736450195 (20.346817184791785) + | > loss_duration: 1.3543466329574585 (1.3430155853393155) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.011539459228516 (30.821169318222417) + | > grad_norm_1: tensor(1064.4150, device='cuda:0') (tensor(1067.3228, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 1.2134 (0.925535499490448) + | > loader_time: 0.0238 (0.01500661780194538) + + + --> TIME: 2024-06-25 13:01:34 -- STEP: 558/591 -- GLOBAL_STEP: 37200 + | > loss_disc: 2.7075746059417725 (2.6255496201976687) + | > loss_disc_real_0: 0.21190917491912842 (0.19700641756523474) + | > loss_disc_real_1: 0.22086872160434723 (0.22331485825200237) + | > loss_disc_real_2: 0.21461549401283264 (0.23108903839383074) + | > loss_disc_real_3: 0.22621718049049377 (0.23321858099177747) + | > loss_disc_real_4: 0.18057315051555634 (0.2108203133427968) + | > loss_disc_real_5: 0.25772231817245483 (0.23791675728922676) + | > loss_0: 2.7075746059417725 (2.6255496201976687) + | > grad_norm_0: tensor(7.9629, device='cuda:0') (tensor(27.2581, device='cuda:0')) + | > loss_gen: 1.9808263778686523 (2.09662458909455) + | > loss_kl: 2.372467041015625 (2.486553213929618) + | > loss_feat: 4.143241882324219 (4.542098089358287) + | > loss_mel: 19.579368591308594 (20.33363009011874) + | > loss_duration: 1.3499414920806885 (1.3446273060255158) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.425846099853516 (30.803533256694834) + | > grad_norm_1: tensor(661.7951, device='cuda:0') (tensor(1066.8645, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 1.1964 (0.9388286546139735) + | > loader_time: 0.0259 (0.015361940134383438) + + + --> TIME: 2024-06-25 13:02:06 -- STEP: 583/591 -- GLOBAL_STEP: 37225 + | > loss_disc: 2.832197427749634 (2.624923801340332) + | > loss_disc_real_0: 0.1905771791934967 (0.19713850917476738) + | > loss_disc_real_1: 0.2274363487958908 (0.22347023076095063) + | > loss_disc_real_2: 0.2196751832962036 (0.23123230801745098) + | > loss_disc_real_3: 0.22768689692020416 (0.23320425397638186) + | > loss_disc_real_4: 0.13303929567337036 (0.20971228883922813) + | > loss_disc_real_5: 0.22790920734405518 (0.2378278392563553) + | > loss_0: 2.832197427749634 (2.624923801340332) + | > grad_norm_0: tensor(12.4428, device='cuda:0') (tensor(27.4851, device='cuda:0')) + | > loss_gen: 2.019895076751709 (2.1024989554239975) + | > loss_kl: 2.4393088817596436 (2.4859796805406322) + | > loss_feat: 4.305233478546143 (4.547009781335273) + | > loss_mel: 19.858470916748047 (20.33113057576567) + | > loss_duration: 1.3842394351959229 (1.3462664829519158) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.00714683532715 (30.81288544146111) + | > grad_norm_1: tensor(76.5905, device='cuda:0') (tensor(1056.2152, device='cuda:0')) + | > current_lr_0: 0.00019845589462876104 + | > current_lr_1: 0.00019845589462876104 + | > step_time: 1.2802 (0.9530747178079334) + | > loader_time: 0.0226 (0.015752403036807857) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.0036053860441167306 (-0.0001252204813855756) + | > avg_loss_disc: 2.601044507736855 (-0.0998938032921326) + | > avg_loss_disc_real_0: 0.19497714270936678 (-0.04474459430004693) + | > avg_loss_disc_real_1: 0.19880375297779732 (+0.0024294824676310556) + | > avg_loss_disc_real_2: 0.20071488935896692 (-0.006620758391441162) + | > avg_loss_disc_real_3: 0.21763839366588186 (+0.028399675133380486) + | > avg_loss_disc_real_4: 0.18018849986664792 (-0.06888581019766787) + | > avg_loss_disc_real_5: 0.2526675526131975 (+0.02632833097843415) + | > avg_loss_0: 2.601044507736855 (-0.0998938032921326) + | > avg_loss_gen: 1.913488989180707 (+0.05838885205857314) + | > avg_loss_kl: 2.5832773726037206 (+0.005922281995732526) + | > avg_loss_feat: 4.367077147707026 (+0.21733395596768013) + | > avg_loss_mel: 20.498896050960465 (-0.32616976474193393) + | > avg_loss_duration: 1.3605386678208695 (-0.0026808824945003806) + | > avg_loss_1: 30.7232783297275 (-0.04720549887798953) + + + > EPOCH: 64/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 13:02:25)  + + --> TIME: 2024-06-25 13:02:40 -- STEP: 17/591 -- GLOBAL_STEP: 37250 + | > loss_disc: 2.702852249145508 (2.643482628990622) + | > loss_disc_real_0: 0.22662386298179626 (0.20608682404546178) + | > loss_disc_real_1: 0.2175653576850891 (0.22592058690155253) + | > loss_disc_real_2: 0.23662161827087402 (0.23649043107734008) + | > loss_disc_real_3: 0.22433552145957947 (0.23409317521487966) + | > loss_disc_real_4: 0.23559564352035522 (0.21579482362550848) + | > loss_disc_real_5: 0.24327999353408813 (0.23707887617980733) + | > loss_0: 2.702852249145508 (2.643482628990622) + | > grad_norm_0: tensor(15.2483, device='cuda:0') (tensor(21.2414, device='cuda:0')) + | > loss_gen: 1.7968649864196777 (2.0929277363945458) + | > loss_kl: 2.698134183883667 (2.632688480265) + | > loss_feat: 4.320709228515625 (4.69657844655654) + | > loss_mel: 20.393518447875977 (20.762578627642462) + | > loss_duration: 1.2393375635147095 (1.3090944921269136) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.448564529418945 (31.493867874145508) + | > grad_norm_1: tensor(322.0786, device='cuda:0') (tensor(972.4204, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.7692 (0.7724695626427146) + | > loader_time: 0.0088 (0.008480142144595875) + + + --> TIME: 2024-06-25 13:03:00 -- STEP: 42/591 -- GLOBAL_STEP: 37275 + | > loss_disc: 2.6130707263946533 (2.648677173114958) + | > loss_disc_real_0: 0.23049497604370117 (0.20362474645177522) + | > loss_disc_real_1: 0.247242733836174 (0.22306757881527856) + | > loss_disc_real_2: 0.23945879936218262 (0.2306643742181006) + | > loss_disc_real_3: 0.2277505099773407 (0.23302857790674483) + | > loss_disc_real_4: 0.23275943100452423 (0.2216166052080336) + | > loss_disc_real_5: 0.23559479415416718 (0.23870302523885453) + | > loss_0: 2.6130707263946533 (2.648677173114958) + | > grad_norm_0: tensor(30.0327, device='cuda:0') (tensor(22.1211, device='cuda:0')) + | > loss_gen: 2.1349565982818604 (2.065902156489236) + | > loss_kl: 2.4920332431793213 (2.5799909148897444) + | > loss_feat: 4.373063564300537 (4.577496222087315) + | > loss_mel: 20.307233810424805 (20.476896195184615) + | > loss_duration: 1.3355835676193237 (1.3169916868209834) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.64286994934082 (31.017277263459704) + | > grad_norm_1: tensor(1376.6443, device='cuda:0') (tensor(988.4421, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.8156 (0.7861582892281669) + | > loader_time: 0.0107 (0.009167500904628206) + + + --> TIME: 2024-06-25 13:03:20 -- STEP: 67/591 -- GLOBAL_STEP: 37300 + | > loss_disc: 2.4600090980529785 (2.620325842900063) + | > loss_disc_real_0: 0.16305986046791077 (0.1996410570260304) + | > loss_disc_real_1: 0.20999445021152496 (0.22262525113660897) + | > loss_disc_real_2: 0.24366170167922974 (0.23032959850866402) + | > loss_disc_real_3: 0.2497021108865738 (0.23225509697821603) + | > loss_disc_real_4: 0.14912059903144836 (0.20907931374524957) + | > loss_disc_real_5: 0.252178430557251 (0.23791448638510349) + | > loss_0: 2.4600090980529785 (2.620325842900063) + | > grad_norm_0: tensor(26.0480, device='cuda:0') (tensor(24.2254, device='cuda:0')) + | > loss_gen: 2.3929121494293213 (2.1100667643902913) + | > loss_kl: 2.5072994232177734 (2.5551526439723684) + | > loss_feat: 5.436971187591553 (4.659175061467868) + | > loss_mel: 20.221097946166992 (20.468243015346243) + | > loss_duration: 1.3405096530914307 (1.323727922653084) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.89879035949707 (31.11636546120715) + | > grad_norm_1: tensor(981.1983, device='cuda:0') (tensor(1027.8412, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.8172 (0.795462839639009) + | > loader_time: 0.0101 (0.009612681260749474) + + + --> TIME: 2024-06-25 13:03:41 -- STEP: 92/591 -- GLOBAL_STEP: 37325 + | > loss_disc: 2.6740477085113525 (2.6204119532004646) + | > loss_disc_real_0: 0.28658002614974976 (0.1990157062916652) + | > loss_disc_real_1: 0.2434626668691635 (0.22318320041117462) + | > loss_disc_real_2: 0.231242835521698 (0.23090614167892415) + | > loss_disc_real_3: 0.23974013328552246 (0.23264516306960065) + | > loss_disc_real_4: 0.21263782680034637 (0.20843820168596247) + | > loss_disc_real_5: 0.21888554096221924 (0.23885980135072832) + | > loss_0: 2.6740477085113525 (2.6204119532004646) + | > grad_norm_0: tensor(22.6804, device='cuda:0') (tensor(27.4502, device='cuda:0')) + | > loss_gen: 1.9610401391983032 (2.1331965158814965) + | > loss_kl: 2.59417462348938 (2.552904795045438) + | > loss_feat: 4.206857681274414 (4.695665193640665) + | > loss_mel: 20.15581703186035 (20.520943289217737) + | > loss_duration: 1.3271455764770508 (1.3307580507319903) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.245033264160156 (31.233467889868695) + | > grad_norm_1: tensor(929.7407, device='cuda:0') (tensor(1016.7188, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.851 (0.8035359875015591) + | > loader_time: 0.013 (0.009929014288860817) + + + --> TIME: 2024-06-25 13:04:03 -- STEP: 117/591 -- GLOBAL_STEP: 37350 + | > loss_disc: 2.6804745197296143 (2.6161935553591475) + | > loss_disc_real_0: 0.26207736134529114 (0.19764301346407998) + | > loss_disc_real_1: 0.22388504445552826 (0.2232987562305907) + | > loss_disc_real_2: 0.24824105203151703 (0.23106166694918248) + | > loss_disc_real_3: 0.23863358795642853 (0.2337099977283396) + | > loss_disc_real_4: 0.2213551551103592 (0.20616201056629166) + | > loss_disc_real_5: 0.24376913905143738 (0.23849134631136543) + | > loss_0: 2.6804745197296143 (2.6161935553591475) + | > grad_norm_0: tensor(30.5177, device='cuda:0') (tensor(27.6492, device='cuda:0')) + | > loss_gen: 2.0366404056549072 (2.135961989052275) + | > loss_kl: 2.444394588470459 (2.5363773015829234) + | > loss_feat: 4.5560102462768555 (4.698969400846039) + | > loss_mel: 20.605201721191406 (20.495513671483746) + | > loss_duration: 1.3059295415878296 (1.3330701457129583) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.94817543029785 (31.199892630943886) + | > grad_norm_1: tensor(1258.9034, device='cuda:0') (tensor(1079.8492, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.843 (0.8103331745180309) + | > loader_time: 0.0116 (0.01019651869423369) + + + --> TIME: 2024-06-25 13:04:24 -- STEP: 142/591 -- GLOBAL_STEP: 37375 + | > loss_disc: 2.6458804607391357 (2.6096154165939547) + | > loss_disc_real_0: 0.18304108083248138 (0.19591084888703386) + | > loss_disc_real_1: 0.2519087493419647 (0.22270234842115724) + | > loss_disc_real_2: 0.24711695313453674 (0.2306807213266131) + | > loss_disc_real_3: 0.2520449757575989 (0.23340526328120434) + | > loss_disc_real_4: 0.15751197934150696 (0.20552871979668108) + | > loss_disc_real_5: 0.23298370838165283 (0.2377673678205047) + | > loss_0: 2.6458804607391357 (2.6096154165939547) + | > grad_norm_0: tensor(34.2165, device='cuda:0') (tensor(28.0605, device='cuda:0')) + | > loss_gen: 1.9451323747634888 (2.1350130631890094) + | > loss_kl: 2.4661102294921875 (2.529907508635185) + | > loss_feat: 4.653090476989746 (4.705732976886586) + | > loss_mel: 20.383222579956055 (20.465382858061453) + | > loss_duration: 1.3417284488677979 (1.3330086931376388) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.789283752441406 (31.16904516623054) + | > grad_norm_1: tensor(791.2971, device='cuda:0') (tensor(1124.0930, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.8425 (0.8170437644904768) + | > loader_time: 0.0113 (0.010480662466774522) + + + --> TIME: 2024-06-25 13:04:46 -- STEP: 167/591 -- GLOBAL_STEP: 37400 + | > loss_disc: 2.714817523956299 (2.6091371610493) + | > loss_disc_real_0: 0.12962861359119415 (0.1953834206044317) + | > loss_disc_real_1: 0.22429530322551727 (0.222405181405787) + | > loss_disc_real_2: 0.2069297879934311 (0.23037801950634595) + | > loss_disc_real_3: 0.25117626786231995 (0.2332363821075348) + | > loss_disc_real_4: 0.19216428697109222 (0.20575288076422169) + | > loss_disc_real_5: 0.2421519160270691 (0.23757143559570085) + | > loss_0: 2.714817523956299 (2.6091371610493) + | > grad_norm_0: tensor(22.3460, device='cuda:0') (tensor(28.3431, device='cuda:0')) + | > loss_gen: 2.0456104278564453 (2.135373876480285) + | > loss_kl: 2.499501943588257 (2.525892964380229) + | > loss_feat: 4.25582218170166 (4.706487053168747) + | > loss_mel: 19.757919311523438 (20.44328113944231) + | > loss_duration: 1.2962895631790161 (1.3313652541109189) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 29.855144500732422 (31.14240036467592) + | > grad_norm_1: tensor(79.9163, device='cuda:0') (tensor(1122.3201, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.8795 (0.8231132772868265) + | > loader_time: 0.013 (0.010694689379480784) + + + --> TIME: 2024-06-25 13:05:08 -- STEP: 192/591 -- GLOBAL_STEP: 37425 + | > loss_disc: 2.671658992767334 (2.6111498412986585) + | > loss_disc_real_0: 0.24822331964969635 (0.1956651599612087) + | > loss_disc_real_1: 0.1987536996603012 (0.2221706317116817) + | > loss_disc_real_2: 0.23509949445724487 (0.23082514386624098) + | > loss_disc_real_3: 0.2278498411178589 (0.23348108058174452) + | > loss_disc_real_4: 0.16956275701522827 (0.20670533572168395) + | > loss_disc_real_5: 0.2188221663236618 (0.23731362400576472) + | > loss_0: 2.671658992767334 (2.6111498412986585) + | > grad_norm_0: tensor(19.3191, device='cuda:0') (tensor(27.8678, device='cuda:0')) + | > loss_gen: 2.0776543617248535 (2.1297325051079192) + | > loss_kl: 2.5485661029815674 (2.520997536679108) + | > loss_feat: 4.713939189910889 (4.693579157193503) + | > loss_mel: 20.981897354125977 (20.439434766769413) + | > loss_duration: 1.3346178531646729 (1.3302806938687963) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.656675338745117 (31.114024708668392) + | > grad_norm_1: tensor(1096.8868, device='cuda:0') (tensor(1112.3015, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.8575 (0.8288976016143957) + | > loader_time: 0.0126 (0.010955004642407099) + + + --> TIME: 2024-06-25 13:05:31 -- STEP: 217/591 -- GLOBAL_STEP: 37450 + | > loss_disc: 2.7079858779907227 (2.612690446563579) + | > loss_disc_real_0: 0.29368263483047485 (0.19621684759305918) + | > loss_disc_real_1: 0.242390438914299 (0.22215704764089278) + | > loss_disc_real_2: 0.2322321981191635 (0.23077327953780302) + | > loss_disc_real_3: 0.28171679377555847 (0.23329893695319304) + | > loss_disc_real_4: 0.2486623376607895 (0.20738958690985004) + | > loss_disc_real_5: 0.2415747195482254 (0.23737562773963822) + | > loss_0: 2.7079858779907227 (2.612690446563579) + | > grad_norm_0: tensor(31.9760, device='cuda:0') (tensor(27.9634, device='cuda:0')) + | > loss_gen: 2.144040822982788 (2.12461429037806) + | > loss_kl: 2.587938070297241 (2.51728583371035) + | > loss_feat: 4.3704094886779785 (4.680206962444817) + | > loss_mel: 20.00858497619629 (20.43223712411345) + | > loss_duration: 1.3007606267929077 (1.3294809144762807) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.411733627319336 (31.083825177311347) + | > grad_norm_1: tensor(1335.0449, device='cuda:0') (tensor(1114.7587, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.8648 (0.8343580823889526) + | > loader_time: 0.0146 (0.011276886759815127) + + + --> TIME: 2024-06-25 13:05:54 -- STEP: 242/591 -- GLOBAL_STEP: 37475 + | > loss_disc: 2.5218467712402344 (2.6115746261659725) + | > loss_disc_real_0: 0.1534419059753418 (0.196286041351143) + | > loss_disc_real_1: 0.20797981321811676 (0.22196597544368635) + | > loss_disc_real_2: 0.20954766869544983 (0.23104844082239245) + | > loss_disc_real_3: 0.21208073198795319 (0.23305659790423291) + | > loss_disc_real_4: 0.1646018624305725 (0.20667762798834435) + | > loss_disc_real_5: 0.23354218900203705 (0.23706439593114143) + | > loss_0: 2.5218467712402344 (2.6115746261659725) + | > grad_norm_0: tensor(23.7025, device='cuda:0') (tensor(28.0953, device='cuda:0')) + | > loss_gen: 2.360142946243286 (2.1266864735232893) + | > loss_kl: 2.504594326019287 (2.5170452013488642) + | > loss_feat: 5.060148239135742 (4.681972413023643) + | > loss_mel: 20.389162063598633 (20.434638646023338) + | > loss_duration: 1.3784067630767822 (1.3295776405610331) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.692455291748047 (31.089920445907214) + | > grad_norm_1: tensor(1581.0183, device='cuda:0') (tensor(1132.2704, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.8802 (0.840793445090617) + | > loader_time: 0.0148 (0.011589953722047412) + + + --> TIME: 2024-06-25 13:06:17 -- STEP: 267/591 -- GLOBAL_STEP: 37500 + | > loss_disc: 2.583554267883301 (2.6128472040655004) + | > loss_disc_real_0: 0.16873228549957275 (0.1961539750744341) + | > loss_disc_real_1: 0.24284057319164276 (0.22192285190360822) + | > loss_disc_real_2: 0.2238917350769043 (0.23180282679613162) + | > loss_disc_real_3: 0.2431599646806717 (0.23306752043716916) + | > loss_disc_real_4: 0.2328605204820633 (0.20701757851946218) + | > loss_disc_real_5: 0.28094470500946045 (0.2375824415728394) + | > loss_0: 2.583554267883301 (2.6128472040655004) + | > grad_norm_0: tensor(29.5150, device='cuda:0') (tensor(28.0492, device='cuda:0')) + | > loss_gen: 2.0705738067626953 (2.123198650302958) + | > loss_kl: 2.525385856628418 (2.515911638959963) + | > loss_feat: 4.393052101135254 (4.668967316659651) + | > loss_mel: 20.362215042114258 (20.414790214224254) + | > loss_duration: 1.360384225845337 (1.3295386194736325) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.711610794067383 (31.052406489625852) + | > grad_norm_1: tensor(1357.6210, device='cuda:0') (tensor(1131.1183, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.8896 (0.8476167475239614) + | > loader_time: 0.0157 (0.011874089080296206) + + + --> TIME: 2024-06-25 13:06:40 -- STEP: 292/591 -- GLOBAL_STEP: 37525 + | > loss_disc: 2.576355218887329 (2.612587830791734) + | > loss_disc_real_0: 0.2021772563457489 (0.1958038778082557) + | > loss_disc_real_1: 0.2724285125732422 (0.22203497364096444) + | > loss_disc_real_2: 0.2513355016708374 (0.23183945546003237) + | > loss_disc_real_3: 0.24017421901226044 (0.23297932014277536) + | > loss_disc_real_4: 0.22573336958885193 (0.2073753263086896) + | > loss_disc_real_5: 0.24405509233474731 (0.2372400162562932) + | > loss_0: 2.576355218887329 (2.612587830791734) + | > grad_norm_0: tensor(36.0308, device='cuda:0') (tensor(28.1357, device='cuda:0')) + | > loss_gen: 2.0108063220977783 (2.121055579348786) + | > loss_kl: 2.559755802154541 (2.51130630055519) + | > loss_feat: 4.70262336730957 (4.663612044020875) + | > loss_mel: 20.62173843383789 (20.40408730833498) + | > loss_duration: 1.3300445079803467 (1.330623564654834) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.22496795654297 (31.030684843455276) + | > grad_norm_1: tensor(1062.2902, device='cuda:0') (tensor(1129.9948, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.9868 (0.8534292603192264) + | > loader_time: 0.0156 (0.012124617622323228) + + + --> TIME: 2024-06-25 13:07:04 -- STEP: 317/591 -- GLOBAL_STEP: 37550 + | > loss_disc: 2.7062435150146484 (2.6119491692973233) + | > loss_disc_real_0: 0.1712542474269867 (0.19528934361900824) + | > loss_disc_real_1: 0.21812810003757477 (0.22215540504982043) + | > loss_disc_real_2: 0.22651976346969604 (0.23173543618116468) + | > loss_disc_real_3: 0.2700735926628113 (0.23322404554401663) + | > loss_disc_real_4: 0.23570546507835388 (0.20648174685916681) + | > loss_disc_real_5: 0.27138155698776245 (0.23732261500712074) + | > loss_0: 2.7062435150146484 (2.6119491692973233) + | > grad_norm_0: tensor(15.8179, device='cuda:0') (tensor(28.2678, device='cuda:0')) + | > loss_gen: 2.1607751846313477 (2.123473642376318) + | > loss_kl: 2.6034843921661377 (2.5089633412165595) + | > loss_feat: 4.298699855804443 (4.664991599904247) + | > loss_mel: 20.18718910217285 (20.41384566394314) + | > loss_duration: 1.327172040939331 (1.331060981148802) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.577320098876953 (31.042335263562126) + | > grad_norm_1: tensor(701.5433, device='cuda:0') (tensor(1126.1652, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.9145 (0.8591324162408004) + | > loader_time: 0.0171 (0.012415843055075272) + + + --> TIME: 2024-06-25 13:07:28 -- STEP: 342/591 -- GLOBAL_STEP: 37575 + | > loss_disc: 2.6808974742889404 (2.614027491089892) + | > loss_disc_real_0: 0.19228380918502808 (0.19543723014798783) + | > loss_disc_real_1: 0.22985941171646118 (0.22203827322575084) + | > loss_disc_real_2: 0.22332358360290527 (0.23171985123241157) + | > loss_disc_real_3: 0.2601124346256256 (0.23302529630256674) + | > loss_disc_real_4: 0.24475276470184326 (0.2079658000049186) + | > loss_disc_real_5: 0.2471868246793747 (0.2373793017968797) + | > loss_0: 2.6808974742889404 (2.614027491089892) + | > grad_norm_0: tensor(8.8997, device='cuda:0') (tensor(27.8841, device='cuda:0')) + | > loss_gen: 2.059690475463867 (2.1169270576789354) + | > loss_kl: 2.5029356479644775 (2.5059491584175526) + | > loss_feat: 4.292858600616455 (4.643704974860474) + | > loss_mel: 20.417343139648438 (20.407881145588842) + | > loss_duration: 1.3601521253585815 (1.332330130345641) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.632980346679688 (31.006792509067825) + | > grad_norm_1: tensor(330.7397, device='cuda:0') (tensor(1109.5261, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.9391 (0.8653451908401578) + | > loader_time: 0.0147 (0.012685754842925485) + + + --> TIME: 2024-06-25 13:07:52 -- STEP: 367/591 -- GLOBAL_STEP: 37600 + | > loss_disc: 2.623199462890625 (2.6147196201927323) + | > loss_disc_real_0: 0.1378643810749054 (0.1953069539664552) + | > loss_disc_real_1: 0.21578390896320343 (0.22192078999342646) + | > loss_disc_real_2: 0.18383795022964478 (0.23168443460068197) + | > loss_disc_real_3: 0.22336748242378235 (0.2329566625304703) + | > loss_disc_real_4: 0.2261059433221817 (0.20903794587226907) + | > loss_disc_real_5: 0.23900939524173737 (0.23720289941379744) + | > loss_0: 2.623199462890625 (2.6147196201927323) + | > grad_norm_0: tensor(38.3172, device='cuda:0') (tensor(27.5991, device='cuda:0')) + | > loss_gen: 2.2977089881896973 (2.1135551292824792) + | > loss_kl: 2.522631883621216 (2.503975308233777) + | > loss_feat: 4.845371246337891 (4.633129813690597) + | > loss_mel: 20.788930892944336 (20.410109621302635) + | > loss_duration: 1.3634215593338013 (1.3340787471802422) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.818063735961914 (30.994848656719324) + | > grad_norm_1: tensor(1483.7738, device='cuda:0') (tensor(1100.6337, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.9554 (0.8713756140311343) + | > loader_time: 0.0176 (0.012961297334052557) + + + --> TIME: 2024-06-25 13:08:17 -- STEP: 392/591 -- GLOBAL_STEP: 37625 + | > loss_disc: 2.53985595703125 (2.6140718137731347) + | > loss_disc_real_0: 0.1412537395954132 (0.19527766716723544) + | > loss_disc_real_1: 0.21173334121704102 (0.2221695207895673) + | > loss_disc_real_2: 0.22102075815200806 (0.2318929063574392) + | > loss_disc_real_3: 0.23484639823436737 (0.2327810900323853) + | > loss_disc_real_4: 0.15478864312171936 (0.20851181263142088) + | > loss_disc_real_5: 0.2558737099170685 (0.237297278505807) + | > loss_0: 2.53985595703125 (2.6140718137731347) + | > grad_norm_0: tensor(36.0439, device='cuda:0') (tensor(27.9293, device='cuda:0')) + | > loss_gen: 2.255941867828369 (2.1156629609818345) + | > loss_kl: 2.460498809814453 (2.501769120595895) + | > loss_feat: 4.874147891998291 (4.631041631406663) + | > loss_mel: 20.456260681152344 (20.39759221368906) + | > loss_duration: 1.3642852306365967 (1.3349504571179955) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.4111328125 (30.98101641693894) + | > grad_norm_1: tensor(1454.7339, device='cuda:0') (tensor(1098.2372, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.993 (0.8778088512469311) + | > loader_time: 0.0172 (0.013217260034716854) + + + --> TIME: 2024-06-25 13:08:43 -- STEP: 417/591 -- GLOBAL_STEP: 37650 + | > loss_disc: 2.5357654094696045 (2.613471156401599) + | > loss_disc_real_0: 0.21192851662635803 (0.19533797855571594) + | > loss_disc_real_1: 0.23767969012260437 (0.2220447282139346) + | > loss_disc_real_2: 0.21312905848026276 (0.2317643597257509) + | > loss_disc_real_3: 0.23135341703891754 (0.2327898068250798) + | > loss_disc_real_4: 0.21276061236858368 (0.20849419949795123) + | > loss_disc_real_5: 0.2549777328968048 (0.23734603500623497) + | > loss_0: 2.5357654094696045 (2.613471156401599) + | > grad_norm_0: tensor(27.6594, device='cuda:0') (tensor(27.7369, device='cuda:0')) + | > loss_gen: 2.1585278511047363 (2.114493891489589) + | > loss_kl: 2.460530996322632 (2.499131901372825) + | > loss_feat: 4.55278205871582 (4.620771132498905) + | > loss_mel: 20.21925926208496 (20.38384555112257) + | > loss_duration: 1.3407573699951172 (1.3358754342694363) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.731857299804688 (30.954117932765605) + | > grad_norm_1: tensor(1009.3486, device='cuda:0') (tensor(1097.7847, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.9905 (0.8849074920590261) + | > loader_time: 0.0182 (0.01349936800894977) + + + --> TIME: 2024-06-25 13:09:08 -- STEP: 442/591 -- GLOBAL_STEP: 37675 + | > loss_disc: 2.5341522693634033 (2.6123152928115005) + | > loss_disc_real_0: 0.21744269132614136 (0.19513513903002935) + | > loss_disc_real_1: 0.21330329775810242 (0.22212513387877478) + | > loss_disc_real_2: 0.22456689178943634 (0.23171412695317248) + | > loss_disc_real_3: 0.19326885044574738 (0.23275814596479294) + | > loss_disc_real_4: 0.19281399250030518 (0.20818614964773746) + | > loss_disc_real_5: 0.2263336181640625 (0.2371826813096914) + | > loss_0: 2.5341522693634033 (2.6123152928115005) + | > grad_norm_0: tensor(28.7767, device='cuda:0') (tensor(28.0918, device='cuda:0')) + | > loss_gen: 1.9502975940704346 (2.1167057663606825) + | > loss_kl: 2.43517804145813 (2.498933617885298) + | > loss_feat: 4.450901985168457 (4.623079444488246) + | > loss_mel: 20.053796768188477 (20.377368573149937) + | > loss_duration: 1.3370858430862427 (1.3367787607654726) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.22726058959961 (30.952866174516636) + | > grad_norm_1: tensor(1177.6709, device='cuda:0') (tensor(1092.8488, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 0.9942 (0.8909142152216639) + | > loader_time: 0.0179 (0.013780958512250112) + + + --> TIME: 2024-06-25 13:09:34 -- STEP: 467/591 -- GLOBAL_STEP: 37700 + | > loss_disc: 2.719385862350464 (2.6164890923387736) + | > loss_disc_real_0: 0.21348753571510315 (0.1955274927092824) + | > loss_disc_real_1: 0.22037678956985474 (0.2223645898910355) + | > loss_disc_real_2: 0.2588144838809967 (0.2318176034941663) + | > loss_disc_real_3: 0.25546929240226746 (0.23297124542694786) + | > loss_disc_real_4: 0.2265109121799469 (0.20960304340471286) + | > loss_disc_real_5: 0.2513180077075958 (0.2372613521153207) + | > loss_0: 2.719385862350464 (2.6164890923387736) + | > grad_norm_0: tensor(11.5095, device='cuda:0') (tensor(27.5653, device='cuda:0')) + | > loss_gen: 1.8546539545059204 (2.1105704049737337) + | > loss_kl: 2.491884469985962 (2.4967745830977006) + | > loss_feat: 4.13796854019165 (4.602984912390369) + | > loss_mel: 20.283361434936523 (20.369401118995285) + | > loss_duration: 1.3500313758850098 (1.338017061129372) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.117900848388672 (30.917748102028824) + | > grad_norm_1: tensor(548.7627, device='cuda:0') (tensor(1066.3336, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 1.0641 (0.8978677270754502) + | > loader_time: 0.0231 (0.01407821122277727) + + + --> TIME: 2024-06-25 13:10:01 -- STEP: 492/591 -- GLOBAL_STEP: 37725 + | > loss_disc: 2.6292684078216553 (2.619684743202798) + | > loss_disc_real_0: 0.2135147899389267 (0.1956294023287975) + | > loss_disc_real_1: 0.22049103677272797 (0.22247396594261734) + | > loss_disc_real_2: 0.25489285588264465 (0.23203332728244425) + | > loss_disc_real_3: 0.2546604573726654 (0.23303155522278654) + | > loss_disc_real_4: 0.20925115048885345 (0.21082335560605286) + | > loss_disc_real_5: 0.24127426743507385 (0.23734474760608945) + | > loss_0: 2.6292684078216553 (2.619684743202798) + | > grad_norm_0: tensor(28.2699, device='cuda:0') (tensor(27.0239, device='cuda:0')) + | > loss_gen: 2.0031304359436035 (2.105053632482279) + | > loss_kl: 2.4896247386932373 (2.4938399210208817) + | > loss_feat: 4.20378303527832 (4.583673320165492) + | > loss_mel: 20.990659713745117 (20.366666429411097) + | > loss_duration: 1.3738746643066406 (1.33953417825505) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.061071395874023 (30.888767540939455) + | > grad_norm_1: tensor(1294.6106, device='cuda:0') (tensor(1038.2186, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 1.0205 (0.9051927536483703) + | > loader_time: 0.0175 (0.014401341356882232) + + + --> TIME: 2024-06-25 13:10:29 -- STEP: 517/591 -- GLOBAL_STEP: 37750 + | > loss_disc: 2.6423861980438232 (2.6214209022558856) + | > loss_disc_real_0: 0.22210857272148132 (0.19567071414431939) + | > loss_disc_real_1: 0.2288065403699875 (0.22265981890016184) + | > loss_disc_real_2: 0.2210370898246765 (0.2323502508826136) + | > loss_disc_real_3: 0.23065729439258575 (0.23280872983904827) + | > loss_disc_real_4: 0.18175488710403442 (0.21119553555430015) + | > loss_disc_real_5: 0.27142617106437683 (0.23734559343562137) + | > loss_0: 2.6423861980438232 (2.6214209022558856) + | > grad_norm_0: tensor(28.0884, device='cuda:0') (tensor(27.1672, device='cuda:0')) + | > loss_gen: 2.161982536315918 (2.102552779869137) + | > loss_kl: 2.3814680576324463 (2.491381579257074) + | > loss_feat: 4.472583770751953 (4.5705350367658575) + | > loss_mel: 20.495094299316406 (20.353128141775805) + | > loss_duration: 1.3864398002624512 (1.3413402305580893) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.897567749023438 (30.85893781872275) + | > grad_norm_1: tensor(1012.0423, device='cuda:0') (tensor(1036.4778, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 1.0997 (0.913800300435817) + | > loader_time: 0.0252 (0.014740844298608065) + + + --> TIME: 2024-06-25 13:10:57 -- STEP: 542/591 -- GLOBAL_STEP: 37775 + | > loss_disc: 2.5382540225982666 (2.621116344339292) + | > loss_disc_real_0: 0.20928074419498444 (0.19568896241403597) + | > loss_disc_real_1: 0.1833195984363556 (0.22277335621883948) + | > loss_disc_real_2: 0.21188147366046906 (0.2324858064801051) + | > loss_disc_real_3: 0.21565458178520203 (0.23281587478859397) + | > loss_disc_real_4: 0.15544459223747253 (0.210316397620853) + | > loss_disc_real_5: 0.20785872638225555 (0.2373580664286314) + | > loss_0: 2.5382540225982666 (2.621116344339292) + | > grad_norm_0: tensor(29.8745, device='cuda:0') (tensor(27.4086, device='cuda:0')) + | > loss_gen: 2.4479336738586426 (2.104640640895743) + | > loss_kl: 2.4457619190216064 (2.4882718263077135) + | > loss_feat: 5.048348903656006 (4.56715786017175) + | > loss_mel: 21.071359634399414 (20.347762709614088) + | > loss_duration: 1.4081088304519653 (1.342963499116722) + | > amp_scaler: 128.0 (66.00738007380075) + | > loss_1: 32.421512603759766 (30.850796586912935) + | > grad_norm_1: tensor(955.0438, device='cuda:0') (tensor(1033.3375, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 1.1074 (0.9236707823742798) + | > loader_time: 0.0242 (0.015070891908174068) + + + --> TIME: 2024-06-25 13:11:28 -- STEP: 567/591 -- GLOBAL_STEP: 37800 + | > loss_disc: 2.6083478927612305 (2.622834999515054) + | > loss_disc_real_0: 0.16286097466945648 (0.19612896473950178) + | > loss_disc_real_1: 0.198297381401062 (0.2229462025762656) + | > loss_disc_real_2: 0.2467045933008194 (0.23256426487229703) + | > loss_disc_real_3: 0.2152080088853836 (0.23304912863164562) + | > loss_disc_real_4: 0.23534870147705078 (0.21040121715588123) + | > loss_disc_real_5: 0.2485014647245407 (0.23743337243006968) + | > loss_0: 2.6083478927612305 (2.622834999515054) + | > grad_norm_0: tensor(45.0808, device='cuda:0') (tensor(27.4286, device='cuda:0')) + | > loss_gen: 2.2092323303222656 (2.1034826166415326) + | > loss_kl: 2.3978545665740967 (2.4858043193817156) + | > loss_feat: 4.201867580413818 (4.554465013416565) + | > loss_mel: 20.380035400390625 (20.331771023059012) + | > loss_duration: 1.3997726440429688 (1.3445801739129237) + | > amp_scaler: 64.0 (66.14462081128747) + | > loss_1: 30.588762283325195 (30.82010320633177) + | > grad_norm_1: tensor(1449.4509, device='cuda:0') (tensor(1030.0072, device='cuda:0')) + | > current_lr_0: 0.00019843108764193245 + | > current_lr_1: 0.00019843108764193245 + | > step_time: 1.1877 (0.9349815403973615) + | > loader_time: 0.0249 (0.015405765075952704) + + + > EVALUATION  + + + --> EVAL PERFORMANCE + | > avg_loader_time: 0.003581919568650266 (-2.3466475466464576e-05) + | > avg_loss_disc: 2.7167088630351612 (+0.1156643552983061) + | > avg_loss_disc_real_0: 0.12730978873181847 (-0.0676673539775483) + | > avg_loss_disc_real_1: 0.1977889189060698 (-0.001014834071727505) + | > avg_loss_disc_real_2: 0.2412709780830018 (+0.04055608872403488) + | > avg_loss_disc_real_3: 0.17635561684344678 (-0.041282776822435086) + | > avg_loss_disc_real_4: 0.21242556958756548 (+0.032237069720917555) + | > avg_loss_disc_real_5: 0.22146654985052475 (-0.031201002762672764) + | > avg_loss_0: 2.7167088630351612 (+0.1156643552983061) + | > avg_loss_gen: 1.685569999065805 (-0.2279189901149019) + | > avg_loss_kl: 2.5303124519104654 (-0.0529649206932552) + | > avg_loss_feat: 4.258404488259172 (-0.10867265944785398) + | > avg_loss_mel: 20.22323547525609 (-0.2756605757043751) + | > avg_loss_duration: 1.3617036012893027 (+0.0011649334684331603) + | > avg_loss_1: 30.059226096944606 (-0.6640522327828933) + + + > EPOCH: 65/1000 + --> /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000 + + > TRAINING (2024-06-25 13:12:06)  + + --> TIME: 2024-06-25 13:12:08 -- STEP: 1/591 -- GLOBAL_STEP: 37825 + | > loss_disc: 2.585416316986084 (2.585416316986084) + | > loss_disc_real_0: 0.26836034655570984 (0.26836034655570984) + | > loss_disc_real_1: 0.2634781301021576 (0.2634781301021576) + | > loss_disc_real_2: 0.2384704202413559 (0.2384704202413559) + | > loss_disc_real_3: 0.2659926116466522 (0.2659926116466522) + | > loss_disc_real_4: 0.24386976659297943 (0.24386976659297943) + | > loss_disc_real_5: 0.22276301681995392 (0.22276301681995392) + | > loss_0: 2.585416316986084 (2.585416316986084) + | > grad_norm_0: tensor(56.7597, device='cuda:0') (tensor(56.7597, device='cuda:0')) + | > loss_gen: 2.372560739517212 (2.372560739517212) + | > loss_kl: 2.6516242027282715 (2.6516242027282715) + | > loss_feat: 5.1741719245910645 (5.1741719245910645) + | > loss_mel: 20.561481475830078 (20.561481475830078) + | > loss_duration: 1.3289453983306885 (1.3289453983306885) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.088783264160156 (32.088783264160156) + | > grad_norm_1: tensor(1340.0160, device='cuda:0') (tensor(1340.0160, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.7522 (0.7522096633911133) + | > loader_time: 0.0088 (0.008780479431152344) + + + --> TIME: 2024-06-25 13:12:28 -- STEP: 26/591 -- GLOBAL_STEP: 37850 + | > loss_disc: 2.6258654594421387 (2.592291694421035) + | > loss_disc_real_0: 0.2236391305923462 (0.1940871522976802) + | > loss_disc_real_1: 0.2350829839706421 (0.22514352374351942) + | > loss_disc_real_2: 0.22316762804985046 (0.2314521515598664) + | > loss_disc_real_3: 0.23487408459186554 (0.23533888046558088) + | > loss_disc_real_4: 0.20810000598430634 (0.19837269473534364) + | > loss_disc_real_5: 0.2570137083530426 (0.24000240919681695) + | > loss_0: 2.6258654594421387 (2.592291694421035) + | > grad_norm_0: tensor(25.1122, device='cuda:0') (tensor(30.1460, device='cuda:0')) + | > loss_gen: 2.1926238536834717 (2.179961846424983) + | > loss_kl: 2.562760591506958 (2.5633934094355664) + | > loss_feat: 4.59280252456665 (4.906888723373414) + | > loss_mel: 20.16034507751465 (20.468005106999325) + | > loss_duration: 1.2676985263824463 (1.3062662436411931) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.776229858398438 (31.424515650822567) + | > grad_norm_1: tensor(1175.2855, device='cuda:0') (tensor(1137.9106, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.803 (0.7757225586817815) + | > loader_time: 0.0092 (0.008921623229980469) + + + --> TIME: 2024-06-25 13:12:48 -- STEP: 51/591 -- GLOBAL_STEP: 37875 + | > loss_disc: 2.6041431427001953 (2.5997680495767033) + | > loss_disc_real_0: 0.14116138219833374 (0.19822179219302008) + | > loss_disc_real_1: 0.20899704098701477 (0.22334093439812755) + | > loss_disc_real_2: 0.2548730671405792 (0.2306602153123594) + | > loss_disc_real_3: 0.25701236724853516 (0.23390650953732284) + | > loss_disc_real_4: 0.1795337200164795 (0.19949595893130584) + | > loss_disc_real_5: 0.26076585054397583 (0.23818843855577357) + | > loss_0: 2.6041431427001953 (2.5997680495767033) + | > grad_norm_0: tensor(28.9411, device='cuda:0') (tensor(29.2934, device='cuda:0')) + | > loss_gen: 2.2734551429748535 (2.162832830466477) + | > loss_kl: 2.5498063564300537 (2.5403244121401927) + | > loss_feat: 4.422872543334961 (4.830094028921689) + | > loss_mel: 19.888154983520508 (20.51978604933795) + | > loss_duration: 1.3306427001953125 (1.3200764328825707) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.46493148803711 (31.373114080990064) + | > grad_norm_1: tensor(1417.0667, device='cuda:0') (tensor(1172.8418, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.8015 (0.7911538002537746) + | > loader_time: 0.0098 (0.009251870361028935) + + + --> TIME: 2024-06-25 13:13:09 -- STEP: 76/591 -- GLOBAL_STEP: 37900 + | > loss_disc: 2.754307746887207 (2.5998752524978235) + | > loss_disc_real_0: 0.16894061863422394 (0.19696928266631927) + | > loss_disc_real_1: 0.24730099737644196 (0.22344980133991493) + | > loss_disc_real_2: 0.2658114731311798 (0.23026499426678607) + | > loss_disc_real_3: 0.2449631243944168 (0.233641536024056) + | > loss_disc_real_4: 0.135276660323143 (0.20000509721668144) + | > loss_disc_real_5: 0.2674641013145447 (0.23774435096665433) + | > loss_0: 2.754307746887207 (2.5998752524978235) + | > grad_norm_0: tensor(29.9223, device='cuda:0') (tensor(31.5569, device='cuda:0')) + | > loss_gen: 2.1281533241271973 (2.1738473898486093) + | > loss_kl: 2.3761916160583496 (2.517853699232403) + | > loss_feat: 4.73427152633667 (4.830652713775635) + | > loss_mel: 21.076154708862305 (20.501240931059186) + | > loss_duration: 1.3284755945205688 (1.324223181134776) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.643247604370117 (31.347818048376787) + | > grad_norm_1: tensor(1353.5370, device='cuda:0') (tensor(1179.3700, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.8092 (0.8000074562273527) + | > loader_time: 0.0107 (0.009682843559666686) + + + --> TIME: 2024-06-25 13:13:30 -- STEP: 101/591 -- GLOBAL_STEP: 37925 + | > loss_disc: 2.6755213737487793 (2.6133933232562376) + | > loss_disc_real_0: 0.11022668331861496 (0.19623880169474248) + | > loss_disc_real_1: 0.20460571348667145 (0.22353507401329456) + | > loss_disc_real_2: 0.21965135633945465 (0.23177063656915534) + | > loss_disc_real_3: 0.23054879903793335 (0.23339629586380306) + | > loss_disc_real_4: 0.22883588075637817 (0.2053164174356083) + | > loss_disc_real_5: 0.2493782639503479 (0.23853987411107166) + | > loss_0: 2.6755213737487793 (2.6133933232562376) + | > grad_norm_0: tensor(19.3199, device='cuda:0') (tensor(29.0061, device='cuda:0')) + | > loss_gen: 2.0821051597595215 (2.1428111307691826) + | > loss_kl: 2.553713083267212 (2.509699254932971) + | > loss_feat: 4.499349594116211 (4.75022849470082) + | > loss_mel: 20.11701011657715 (20.489500933354442) + | > loss_duration: 1.3438199758529663 (1.3315529256764027) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.595998764038086 (31.22379282205412) + | > grad_norm_1: tensor(555.8421, device='cuda:0') (tensor(1072.7625, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.8248 (0.8080161373213967) + | > loader_time: 0.0122 (0.009999610409878266) + + + --> TIME: 2024-06-25 13:13:52 -- STEP: 126/591 -- GLOBAL_STEP: 37950 + | > loss_disc: 2.6436378955841064 (2.6200703420336287) + | > loss_disc_real_0: 0.17642012238502502 (0.1964349882115447) + | > loss_disc_real_1: 0.20304568111896515 (0.22339713088576757) + | > loss_disc_real_2: 0.2533782720565796 (0.23186090115516905) + | > loss_disc_real_3: 0.23835508525371552 (0.23341661502444555) + | > loss_disc_real_4: 0.2400510013103485 (0.20939454057859996) + | > loss_disc_real_5: 0.22181592881679535 (0.23803702150545422) + | > loss_0: 2.6436378955841064 (2.6200703420336287) + | > grad_norm_0: tensor(20.4853, device='cuda:0') (tensor(27.6413, device='cuda:0')) + | > loss_gen: 2.0323972702026367 (2.1230510151575483) + | > loss_kl: 2.5824851989746094 (2.4997246946607325) + | > loss_feat: 4.637090682983398 (4.704032667099484) + | > loss_mel: 20.561979293823242 (20.472687388223314) + | > loss_duration: 1.3423519134521484 (1.333495048303453) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.15630531311035 (31.13299088251023) + | > grad_norm_1: tensor(1005.9315, device='cuda:0') (tensor(1040.3262, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.844 (0.8146616731371198) + | > loader_time: 0.0111 (0.010237924636356413) + + + --> TIME: 2024-06-25 13:14:14 -- STEP: 151/591 -- GLOBAL_STEP: 37975 + | > loss_disc: 2.716930866241455 (2.6180094962088485) + | > loss_disc_real_0: 0.1341772973537445 (0.1959722799477198) + | > loss_disc_real_1: 0.1996336281299591 (0.2229700553298786) + | > loss_disc_real_2: 0.1976330727338791 (0.23130388291466314) + | > loss_disc_real_3: 0.2006862312555313 (0.23297946105729664) + | > loss_disc_real_4: 0.15276533365249634 (0.20930317636357237) + | > loss_disc_real_5: 0.2147592455148697 (0.2383248604686055) + | > loss_0: 2.716930866241455 (2.6180094962088485) + | > grad_norm_0: tensor(18.1601, device='cuda:0') (tensor(27.6549, device='cuda:0')) + | > loss_gen: 2.1134297847747803 (2.126615270873569) + | > loss_kl: 2.4099252223968506 (2.4973938765115298) + | > loss_feat: 4.515281677246094 (4.720409074366486) + | > loss_mel: 20.11829376220703 (20.492980957031246) + | > loss_duration: 1.3093912601470947 (1.3321339239347854) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.46632194519043 (31.16953319271669) + | > grad_norm_1: tensor(651.0989, device='cuda:0') (tensor(1047.7631, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.8592 (0.8209245299661398) + | > loader_time: 0.0116 (0.010504349967501811) + + + --> TIME: 2024-06-25 13:14:36 -- STEP: 176/591 -- GLOBAL_STEP: 38000 + | > loss_disc: 2.6024770736694336 (2.6137448562817145) + | > loss_disc_real_0: 0.23979750275611877 (0.19579907536337318) + | > loss_disc_real_1: 0.19283778965473175 (0.22352643565020777) + | > loss_disc_real_2: 0.23806068301200867 (0.23126365218988873) + | > loss_disc_real_3: 0.2490769326686859 (0.23300885417583314) + | > loss_disc_real_4: 0.20698006451129913 (0.20712508718398484) + | > loss_disc_real_5: 0.21178998053073883 (0.23801481910049915) + | > loss_0: 2.6024770736694336 (2.6137448562817145) + | > grad_norm_0: tensor(26.1995, device='cuda:0') (tensor(28.3361, device='cuda:0')) + | > loss_gen: 2.1008825302124023 (2.136281639337539) + | > loss_kl: 2.5120694637298584 (2.4957398514855993) + | > loss_feat: 4.853428363800049 (4.736075165596875) + | > loss_mel: 19.920080184936523 (20.4703181331808) + | > loss_duration: 1.2974982261657715 (1.3302037492394452) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.683958053588867 (31.168618603186175) + | > grad_norm_1: tensor(1280.1060, device='cuda:0') (tensor(1049.0461, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.8852 (0.8267326029864228) + | > loader_time: 0.0123 (0.010804855010726231) + + + --> TIME: 2024-06-25 13:14:58 -- STEP: 201/591 -- GLOBAL_STEP: 38025 + | > loss_disc: 2.533747673034668 (2.6120441908860084) + | > loss_disc_real_0: 0.18701712787151337 (0.19531367159453195) + | > loss_disc_real_1: 0.22348231077194214 (0.2236350315868558) + | > loss_disc_real_2: 0.2491469383239746 (0.23118291773013214) + | > loss_disc_real_3: 0.26619791984558105 (0.23325881992109973) + | > loss_disc_real_4: 0.234597310423851 (0.207138491877869) + | > loss_disc_real_5: 0.25352463126182556 (0.23773657593561048) + | > loss_0: 2.533747673034668 (2.6120441908860084) + | > grad_norm_0: tensor(30.6932, device='cuda:0') (tensor(28.0422, device='cuda:0')) + | > loss_gen: 2.1538712978363037 (2.1369370583871112) + | > loss_kl: 2.501952886581421 (2.492369574693897) + | > loss_feat: 4.793626308441162 (4.735180081419686) + | > loss_mel: 20.85359764099121 (20.48122732437666) + | > loss_duration: 1.2940508127212524 (1.329327313461115) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.59709930419922 (31.175041435962886) + | > grad_norm_1: tensor(1128.3531, device='cuda:0') (tensor(1051.6832, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.8754 (0.832346110794675) + | > loader_time: 0.0145 (0.011075544120067388) + + + --> TIME: 2024-06-25 13:15:20 -- STEP: 226/591 -- GLOBAL_STEP: 38050 + | > loss_disc: 2.522799253463745 (2.612794923571358) + | > loss_disc_real_0: 0.19846776127815247 (0.19586318566472127) + | > loss_disc_real_1: 0.20928964018821716 (0.2231145603335009) + | > loss_disc_real_2: 0.23935383558273315 (0.23113470920155535) + | > loss_disc_real_3: 0.21697133779525757 (0.23293726851718616) + | > loss_disc_real_4: 0.17968575656414032 (0.20690542360586403) + | > loss_disc_real_5: 0.24090316891670227 (0.2378198656756266) + | > loss_0: 2.522799253463745 (2.612794923571358) + | > grad_norm_0: tensor(21.8622, device='cuda:0') (tensor(28.2959, device='cuda:0')) + | > loss_gen: 2.3787646293640137 (2.1350492149327702) + | > loss_kl: 2.548431634902954 (2.4950336555464068) + | > loss_feat: 4.925296306610107 (4.725585213804668) + | > loss_mel: 20.875688552856445 (20.474200181201503) + | > loss_duration: 1.2961325645446777 (1.3284371505796393) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 32.024314880371094 (31.158305522614874) + | > grad_norm_1: tensor(1756.8435, device='cuda:0') (tensor(1062.4548, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.8853 (0.8376483674597953) + | > loader_time: 0.0152 (0.011403630265092429) + + + --> TIME: 2024-06-25 13:15:43 -- STEP: 251/591 -- GLOBAL_STEP: 38075 + | > loss_disc: 2.5042121410369873 (2.6109603192226816) + | > loss_disc_real_0: 0.22720131278038025 (0.19593537854127197) + | > loss_disc_real_1: 0.22468098998069763 (0.22279968394701224) + | > loss_disc_real_2: 0.22146016359329224 (0.23083539479282272) + | > loss_disc_real_3: 0.20754627883434296 (0.23284002503313392) + | > loss_disc_real_4: 0.19708433747291565 (0.20677788021317517) + | > loss_disc_real_5: 0.22476144134998322 (0.23750214266824532) + | > loss_0: 2.5042121410369873 (2.6109603192226816) + | > grad_norm_0: tensor(25.5961, device='cuda:0') (tensor(28.3848, device='cuda:0')) + | > loss_gen: 2.251497745513916 (2.133399409601887) + | > loss_kl: 2.37164568901062 (2.4940020439634276) + | > loss_feat: 5.0517730712890625 (4.71899585799867) + | > loss_mel: 20.402908325195312 (20.46167125929875) + | > loss_duration: 1.3226573467254639 (1.328869511406736) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.400482177734375 (31.136938171082758) + | > grad_norm_1: tensor(1297.8712, device='cuda:0') (tensor(1075.5424, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.8955 (0.8427807686338389) + | > loader_time: 0.0147 (0.011703402872579507) + + + --> TIME: 2024-06-25 13:16:06 -- STEP: 276/591 -- GLOBAL_STEP: 38100 + | > loss_disc: 2.6046175956726074 (2.61031581958135) + | > loss_disc_real_0: 0.19158780574798584 (0.19535412453114986) + | > loss_disc_real_1: 0.23902039229869843 (0.22278866614552512) + | > loss_disc_real_2: 0.2537364065647125 (0.23079945163234417) + | > loss_disc_real_3: 0.20179246366024017 (0.23278571535711703) + | > loss_disc_real_4: 0.2396661937236786 (0.20670032425635104) + | > loss_disc_real_5: 0.22805896401405334 (0.23780331425908682) + | > loss_0: 2.6046175956726074 (2.61031581958135) + | > grad_norm_0: tensor(30.3663, device='cuda:0') (tensor(28.3982, device='cuda:0')) + | > loss_gen: 2.200774908065796 (2.1323215697986484) + | > loss_kl: 2.465827703475952 (2.493059163508208) + | > loss_feat: 4.837940692901611 (4.713253520537114) + | > loss_mel: 20.899065017700195 (20.446956302808687) + | > loss_duration: 1.3413338661193848 (1.3290743689606164) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.74494171142578 (31.114664989969004) + | > grad_norm_1: tensor(1409.5760, device='cuda:0') (tensor(1081.3936, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.8913 (0.8481217147647473) + | > loader_time: 0.0136 (0.011955804582955181) + + + --> TIME: 2024-06-25 13:16:29 -- STEP: 301/591 -- GLOBAL_STEP: 38125 + | > loss_disc: 2.7182157039642334 (2.6113112923314805) + | > loss_disc_real_0: 0.243108868598938 (0.19601015447085082) + | > loss_disc_real_1: 0.17712780833244324 (0.22263941223241165) + | > loss_disc_real_2: 0.25054609775543213 (0.23091639315963183) + | > loss_disc_real_3: 0.24026283621788025 (0.23281041277801476) + | > loss_disc_real_4: 0.1909704953432083 (0.20627813531314812) + | > loss_disc_real_5: 0.23619522154331207 (0.23774285852117) + | > loss_0: 2.7182157039642334 (2.6113112923314805) + | > grad_norm_0: tensor(15.6562, device='cuda:0') (tensor(28.7503, device='cuda:0')) + | > loss_gen: 1.9487851858139038 (2.134006923615338) + | > loss_kl: 2.5109260082244873 (2.4915526412254145) + | > loss_feat: 4.723555088043213 (4.714846031214314) + | > loss_mel: 20.585968017578125 (20.443725953466483) + | > loss_duration: 1.3536667823791504 (1.3294890212062194) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.122901916503906 (31.11362063132251) + | > grad_norm_1: tensor(837.8876, device='cuda:0') (tensor(1085.3633, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.9041 (0.8537230753027327) + | > loader_time: 0.016 (0.012211064563637164) + + + --> TIME: 2024-06-25 13:16:53 -- STEP: 326/591 -- GLOBAL_STEP: 38150 + | > loss_disc: 2.548473358154297 (2.6101598637235677) + | > loss_disc_real_0: 0.25489944219589233 (0.19596300637520883) + | > loss_disc_real_1: 0.22370503842830658 (0.22306005396177433) + | > loss_disc_real_2: 0.23404259979724884 (0.23093683766258272) + | > loss_disc_real_3: 0.21993838250637054 (0.2327539928577429) + | > loss_disc_real_4: 0.18983611464500427 (0.2056097644238384) + | > loss_disc_real_5: 0.23297376930713654 (0.23762461611646815) + | > loss_0: 2.548473358154297 (2.6101598637235677) + | > grad_norm_0: tensor(30.7687, device='cuda:0') (tensor(28.9009, device='cuda:0')) + | > loss_gen: 2.071087121963501 (2.133295673899855) + | > loss_kl: 2.395622968673706 (2.490275098502269) + | > loss_feat: 4.9322967529296875 (4.707484110732753) + | > loss_mel: 20.344133377075195 (20.438916165404535) + | > loss_duration: 1.3503611087799072 (1.3306988353378202) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.093502044677734 (31.10066993689976) + | > grad_norm_1: tensor(1531.3281, device='cuda:0') (tensor(1102.3196, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.9279 (0.8596336293074255) + | > loader_time: 0.0163 (0.012440560785539314) + + + --> TIME: 2024-06-25 13:17:18 -- STEP: 351/591 -- GLOBAL_STEP: 38175 + | > loss_disc: 2.5537827014923096 (2.609787341875908) + | > loss_disc_real_0: 0.15505489706993103 (0.19566317838736058) + | > loss_disc_real_1: 0.2568098306655884 (0.22312183457392235) + | > loss_disc_real_2: 0.22780193388462067 (0.23100993376851417) + | > loss_disc_real_3: 0.20766402781009674 (0.23260350431981588) + | > loss_disc_real_4: 0.1825011521577835 (0.20572621199140523) + | > loss_disc_real_5: 0.2291841208934784 (0.23762576112210582) + | > loss_0: 2.5537827014923096 (2.609787341875908) + | > grad_norm_0: tensor(27.6721, device='cuda:0') (tensor(29.1680, device='cuda:0')) + | > loss_gen: 2.2231128215789795 (2.131453388776534) + | > loss_kl: 2.4408674240112305 (2.4889012870625535) + | > loss_feat: 4.5232720375061035 (4.696086989508736) + | > loss_mel: 19.728870391845703 (20.41728852209543) + | > loss_duration: 1.2994985580444336 (1.3319026564600802) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.215621948242188 (31.065632882620875) + | > grad_norm_1: tensor(1484.2998, device='cuda:0') (tensor(1114.3071, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 1.0123 (0.8665289192797455) + | > loader_time: 0.0168 (0.012678249609096779) + + + --> TIME: 2024-06-25 13:17:42 -- STEP: 376/591 -- GLOBAL_STEP: 38200 + | > loss_disc: 2.74023699760437 (2.6117910298895337) + | > loss_disc_real_0: 0.23542596399784088 (0.19576063480703743) + | > loss_disc_real_1: 0.2689109742641449 (0.22309439502497938) + | > loss_disc_real_2: 0.27597421407699585 (0.23109016979628416) + | > loss_disc_real_3: 0.289996862411499 (0.23308798520171897) + | > loss_disc_real_4: 0.18376897275447845 (0.20594856939575773) + | > loss_disc_real_5: 0.26848500967025757 (0.23762947923325478) + | > loss_0: 2.74023699760437 (2.6117910298895337) + | > grad_norm_0: tensor(27.6282, device='cuda:0') (tensor(29.4638, device='cuda:0')) + | > loss_gen: 1.9793872833251953 (2.1293390250586457) + | > loss_kl: 2.4787650108337402 (2.489222726923352) + | > loss_feat: 4.144300937652588 (4.68575852221631) + | > loss_mel: 20.247329711914062 (20.403163189583648) + | > loss_duration: 1.3771908283233643 (1.3334409749888363) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.226974487304688 (31.040924462866275) + | > grad_norm_1: tensor(1126.3011, device='cuda:0') (tensor(1122.4938, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.9512 (0.8728014718978966) + | > loader_time: 0.0183 (0.01299582326665838) + + + --> TIME: 2024-06-25 13:18:09 -- STEP: 401/591 -- GLOBAL_STEP: 38225 + | > loss_disc: 2.677292585372925 (2.613430057083283) + | > loss_disc_real_0: 0.16843801736831665 (0.19614211566504683) + | > loss_disc_real_1: 0.24786408245563507 (0.22310771378793026) + | > loss_disc_real_2: 0.26136383414268494 (0.23106750303373072) + | > loss_disc_real_3: 0.23926512897014618 (0.23306524285354518) + | > loss_disc_real_4: 0.2069120705127716 (0.20636328118400382) + | > loss_disc_real_5: 0.22151705622673035 (0.23776422169737685) + | > loss_0: 2.677292585372925 (2.613430057083283) + | > grad_norm_0: tensor(34.5185, device='cuda:0') (tensor(29.5963, device='cuda:0')) + | > loss_gen: 1.9264862537384033 (2.1256294761809946) + | > loss_kl: 2.5609335899353027 (2.4888351141960525) + | > loss_feat: 4.263799667358398 (4.66970680836133) + | > loss_mel: 20.335878372192383 (20.395354482598446) + | > loss_duration: 1.3561506271362305 (1.3345292996884588) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.443248748779297 (31.01405519024095) + | > grad_norm_1: tensor(782.7834, device='cuda:0') (tensor(1129.6980, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.9595 (0.8829882632467221) + | > loader_time: 0.0166 (0.013779794188806246) + + + --> TIME: 2024-06-25 13:18:34 -- STEP: 426/591 -- GLOBAL_STEP: 38250 + | > loss_disc: 2.5909993648529053 (2.6129698988417513) + | > loss_disc_real_0: 0.23412539064884186 (0.19635279191561697) + | > loss_disc_real_1: 0.22064219415187836 (0.22287821059635549) + | > loss_disc_real_2: 0.22078555822372437 (0.23087520666525393) + | > loss_disc_real_3: 0.22777874767780304 (0.23295255526550498) + | > loss_disc_real_4: 0.20016133785247803 (0.20612947223052172) + | > loss_disc_real_5: 0.22266137599945068 (0.2377462237373764) + | > loss_0: 2.5909993648529053 (2.6129698988417513) + | > grad_norm_0: tensor(34.8922, device='cuda:0') (tensor(29.7113, device='cuda:0')) + | > loss_gen: 2.0608017444610596 (2.124489589196418) + | > loss_kl: 2.422098398208618 (2.4891456751756245) + | > loss_feat: 4.690544605255127 (4.662774562835692) + | > loss_mel: 20.59807014465332 (20.39864610842137) + | > loss_duration: 1.3850656747817993 (1.3356763261584614) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.156579971313477 (31.01073226122789) + | > grad_norm_1: tensor(1758.6206, device='cuda:0') (tensor(1136.6078, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 0.9797 (0.8894697539683242) + | > loader_time: 0.0175 (0.014020903569431933) + + + --> TIME: 2024-06-25 13:19:00 -- STEP: 451/591 -- GLOBAL_STEP: 38275 + | > loss_disc: 2.623584747314453 (2.6157340622794085) + | > loss_disc_real_0: 0.23828835785388947 (0.19661459520029642) + | > loss_disc_real_1: 0.2280508428812027 (0.22300408017344592) + | > loss_disc_real_2: 0.245047926902771 (0.2310341035910033) + | > loss_disc_real_3: 0.22152921557426453 (0.23299697233145095) + | > loss_disc_real_4: 0.22974565625190735 (0.2073607655560098) + | > loss_disc_real_5: 0.2687474191188812 (0.23779551063699364) + | > loss_0: 2.623584747314453 (2.6157340622794085) + | > grad_norm_0: tensor(20.1199, device='cuda:0') (tensor(29.4075, device='cuda:0')) + | > loss_gen: 1.930583119392395 (2.1189627977802603) + | > loss_kl: 2.4393019676208496 (2.4887045666806693) + | > loss_feat: 4.588372707366943 (4.641853402300577) + | > loss_mel: 20.731924057006836 (20.39171513221217) + | > loss_duration: 1.354846477508545 (1.3368058098922024) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 31.045028686523438 (30.97804171441664) + | > grad_norm_1: tensor(818.7327, device='cuda:0') (tensor(1121.8975, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 1.0021 (0.8957847993812648) + | > loader_time: 0.0174 (0.01427785467414264) + + + --> TIME: 2024-06-25 13:19:26 -- STEP: 476/591 -- GLOBAL_STEP: 38300 + | > loss_disc: 2.6588103771209717 (2.616237040327378) + | > loss_disc_real_0: 0.25753992795944214 (0.19644336018930456) + | > loss_disc_real_1: 0.2263222634792328 (0.22292445202459807) + | > loss_disc_real_2: 0.2369077056646347 (0.23085014130763645) + | > loss_disc_real_3: 0.23586633801460266 (0.23303669741173752) + | > loss_disc_real_4: 0.21827204525470734 (0.20784740331543594) + | > loss_disc_real_5: 0.24063751101493835 (0.23785720934637455) + | > loss_0: 2.6588103771209717 (2.616237040327378) + | > grad_norm_0: tensor(32.3378, device='cuda:0') (tensor(29.3645, device='cuda:0')) + | > loss_gen: 2.038808822631836 (2.1164431414183458) + | > loss_kl: 2.467393398284912 (2.4871894977673743) + | > loss_feat: 4.1894755363464355 (4.628601826038679) + | > loss_mel: 20.3717041015625 (20.381302749409404) + | > loss_duration: 1.3515384197235107 (1.3379825293516918) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.41891860961914 (30.951519757759673) + | > grad_norm_1: tensor(954.9583, device='cuda:0') (tensor(1119.7843, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 1.0715 (0.90252130121744) + | > loader_time: 0.02 (0.014532587107490093) + + + --> TIME: 2024-06-25 13:19:54 -- STEP: 501/591 -- GLOBAL_STEP: 38325 + | > loss_disc: 2.6265642642974854 (2.6166812759673532) + | > loss_disc_real_0: 0.25035932660102844 (0.1964808968815974) + | > loss_disc_real_1: 0.25330308079719543 (0.2229306678512615) + | > loss_disc_real_2: 0.25802552700042725 (0.23102818295627295) + | > loss_disc_real_3: 0.27920666337013245 (0.23305183393274714) + | > loss_disc_real_4: 0.21884363889694214 (0.2078070157837725) + | > loss_disc_real_5: 0.26445505023002625 (0.23789865739212301) + | > loss_0: 2.6265642642974854 (2.6166812759673532) + | > grad_norm_0: tensor(27.4215, device='cuda:0') (tensor(29.4364, device='cuda:0')) + | > loss_gen: 2.1554951667785645 (2.1157438728861715) + | > loss_kl: 2.3067920207977295 (2.4853706593047105) + | > loss_feat: 4.391007423400879 (4.621696941866844) + | > loss_mel: 19.78170394897461 (20.373006702659143) + | > loss_duration: 1.3765103816986084 (1.3394827207405415) + | > amp_scaler: 64.0 (64.0) + | > loss_1: 30.01150894165039 (30.935300899360946) + | > grad_norm_1: tensor(1312.5022, device='cuda:0') (tensor(1119.3855, device='cuda:0')) + | > current_lr_0: 0.0001984062837559772 + | > current_lr_1: 0.0001984062837559772 + | > step_time: 1.1407 (0.9109760239690604) + | > loader_time: 0.021 (0.014811383036082378) + + > Keyboard interrupt detected. + > Saving model before exiting... + + > CHECKPOINT : /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000/checkpoint_38348.pth + ! Run is kept in /home/ubuntu/vits-train/vits_nctb-June-25-2024_02+44AM-0000000