Model loaded from checkpoints/CP2-448.pth
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449]

    Starting training:
        Epochs: 3
        Batch size: 1
        Learning rate: 0.1
        Training size: 450
        Validation size: 50
        Checkpoints: True
        CUDA: True
    

    im_tags: ['frame_loose_lf0', 'frame_mp2_roi0', 'frame_mp3_roi0']
    ma_tags: ['frame_ductor0']
    truth_th: 100
    

        file_img: data/cosmic-rec-0.h5
        file_mask: data/cosmic-tru-0.h5
        
Starting epoch 3/3.
0.0000 --- loss: 0.028476
0.0022 --- loss: 0.014019
0.0044 --- loss: 0.015748
0.0067 --- loss: 0.016940
0.0089 --- loss: 0.013026
0.0111 --- loss: 0.010016
0.0133 --- loss: 0.012720
0.0156 --- loss: 0.008137
0.0178 --- loss: 0.008544
0.0200 --- loss: 0.005832
0.0222 --- loss: 0.006796
0.0244 --- loss: 0.022150
0.0267 --- loss: 0.006827
0.0289 --- loss: 0.013690
0.0311 --- loss: 0.010809
0.0333 --- loss: 0.007639
0.0356 --- loss: 0.016921
0.0378 --- loss: 0.009590
0.0400 --- loss: 0.014071
0.0422 --- loss: 0.011413
0.0444 --- loss: 0.012617
0.0467 --- loss: 0.010775
0.0489 --- loss: 0.010407
0.0511 --- loss: 0.011049
0.0533 --- loss: 0.010417
0.0556 --- loss: 0.006906
0.0578 --- loss: 0.008404
0.0600 --- loss: 0.007186
0.0622 --- loss: 0.007325
0.0644 --- loss: 0.005659
0.0667 --- loss: 0.008740
0.0689 --- loss: 0.012266
0.0711 --- loss: 0.016890
0.0733 --- loss: 0.013807
0.0756 --- loss: 0.006250
0.0778 --- loss: 0.010363
0.0800 --- loss: 0.007544
0.0822 --- loss: 0.009699
0.0844 --- loss: 0.013389
0.0867 --- loss: 0.010398
0.0889 --- loss: 0.018801
0.0911 --- loss: 0.004996
0.0933 --- loss: 0.011063
0.0956 --- loss: 0.012027
0.0978 --- loss: 0.005373
0.1000 --- loss: 0.005749
0.1022 --- loss: 0.008999
0.1044 --- loss: 0.010901
0.1067 --- loss: 0.005911
0.1089 --- loss: 0.013431
0.1111 --- loss: 0.006609
0.1133 --- loss: 0.012320
0.1156 --- loss: 0.008641
0.1178 --- loss: 0.004098
0.1200 --- loss: 0.009874
0.1222 --- loss: 0.011744
0.1244 --- loss: 0.011397
0.1267 --- loss: 0.008152
0.1289 --- loss: 0.006450
0.1311 --- loss: 0.008791
0.1333 --- loss: 0.007796
0.1356 --- loss: 0.009432
0.1378 --- loss: 0.017386
0.1400 --- loss: 0.011758
0.1422 --- loss: 0.006829
0.1444 --- loss: 0.005967
0.1467 --- loss: 0.007126
0.1489 --- loss: 0.023014
0.1511 --- loss: 0.008265
0.1533 --- loss: 0.008673
0.1556 --- loss: 0.013369
0.1578 --- loss: 0.007655
0.1600 --- loss: 0.011149
0.1622 --- loss: 0.011945
0.1644 --- loss: 0.005938
0.1667 --- loss: 0.005485
0.1689 --- loss: 0.009630
0.1711 --- loss: 0.013100
0.1733 --- loss: 0.013155
0.1756 --- loss: 0.005181
0.1778 --- loss: 0.008952
0.1800 --- loss: 0.009738
0.1822 --- loss: 0.016328
0.1844 --- loss: 0.008087
0.1867 --- loss: 0.005594
0.1889 --- loss: 0.005399
0.1911 --- loss: 0.016168
0.1933 --- loss: 0.007739
0.1956 --- loss: 0.034438
0.1978 --- loss: 0.011692
0.2000 --- loss: 0.016118
0.2022 --- loss: 0.011114
0.2044 --- loss: 0.005360
0.2067 --- loss: 0.015037
0.2089 --- loss: 0.005944
0.2111 --- loss: 0.008678
0.2133 --- loss: 0.012063
0.2156 --- loss: 0.007434
0.2178 --- loss: 0.010249
0.2200 --- loss: 0.012919
0.2222 --- loss: 0.014889
0.2244 --- loss: 0.004679
0.2267 --- loss: 0.007988
0.2289 --- loss: 0.007604
0.2311 --- loss: 0.012382
0.2333 --- loss: 0.007532
0.2356 --- loss: 0.011714
0.2378 --- loss: 0.006090
0.2400 --- loss: 0.009763
0.2422 --- loss: 0.012836
0.2444 --- loss: 0.007442
0.2467 --- loss: 0.012192
0.2489 --- loss: 0.011127
0.2511 --- loss: 0.010011
0.2533 --- loss: 0.008696
0.2556 --- loss: 0.013756
0.2578 --- loss: 0.016536
0.2600 --- loss: 0.009040
0.2622 --- loss: 0.007152
0.2644 --- loss: 0.006882
0.2667 --- loss: 0.009891
0.2689 --- loss: 0.042984
0.2711 --- loss: 0.005439
0.2733 --- loss: 0.012666
0.2756 --- loss: 0.011194
0.2778 --- loss: 0.009043
0.2800 --- loss: 0.014665
0.2822 --- loss: 0.005498
0.2844 --- loss: 0.009223
0.2867 --- loss: 0.007604
0.2889 --- loss: 0.009232
0.2911 --- loss: 0.011893
0.2933 --- loss: 0.008592
0.2956 --- loss: 0.005742
0.2978 --- loss: 0.008067
0.3000 --- loss: 0.008320
0.3022 --- loss: 0.007618
0.3044 --- loss: 0.010895
0.3067 --- loss: 0.006843
0.3089 --- loss: 0.007823
0.3111 --- loss: 0.007884
0.3133 --- loss: 0.005342
0.3156 --- loss: 0.007902
0.3178 --- loss: 0.007122
0.3200 --- loss: 0.011183
0.3222 --- loss: 0.015985
0.3244 --- loss: 0.009510
0.3267 --- loss: 0.011681
0.3289 --- loss: 0.008051
0.3311 --- loss: 0.008729
0.3333 --- loss: 0.005111
0.3356 --- loss: 0.012373
0.3378 --- loss: 0.010181
0.3400 --- loss: 0.010714
0.3422 --- loss: 0.014009
0.3444 --- loss: 0.012435
0.3467 --- loss: 0.004679
0.3489 --- loss: 0.006949
0.3511 --- loss: 0.008375
0.3533 --- loss: 0.005941
0.3556 --- loss: 0.009596
0.3578 --- loss: 0.010445
0.3600 --- loss: 0.010686
0.3622 --- loss: 0.009787
0.3644 --- loss: 0.006730
0.3667 --- loss: 0.005618
0.3689 --- loss: 0.009459
0.3711 --- loss: 0.008994
0.3733 --- loss: 0.011179
0.3756 --- loss: 0.008095
0.3778 --- loss: 0.004465
0.3800 --- loss: 0.008580
0.3822 --- loss: 0.004591
0.3844 --- loss: 0.005049
0.3867 --- loss: 0.012621
0.3889 --- loss: 0.006494
0.3911 --- loss: 0.008172
0.3933 --- loss: 0.013949
0.3956 --- loss: 0.008194
0.3978 --- loss: 0.010891
0.4000 --- loss: 0.009951
0.4022 --- loss: 0.006531
0.4044 --- loss: 0.006982
0.4067 --- loss: 0.006271
0.4089 --- loss: 0.005216
0.4111 --- loss: 0.006978
0.4133 --- loss: 0.011515
0.4156 --- loss: 0.007366
0.4178 --- loss: 0.006002
0.4200 --- loss: 0.004362
0.4222 --- loss: 0.006282
0.4244 --- loss: 0.018570
0.4267 --- loss: 0.009762
0.4289 --- loss: 0.007440
0.4311 --- loss: 0.008412
0.4333 --- loss: 0.011463
0.4356 --- loss: 0.011959
0.4378 --- loss: 0.010665
0.4400 --- loss: 0.013029
0.4422 --- loss: 0.005883
0.4444 --- loss: 0.010874
0.4467 --- loss: 0.009247
0.4489 --- loss: 0.008669
0.4511 --- loss: 0.007645
0.4533 --- loss: 0.017237
0.4556 --- loss: 0.010662
0.4578 --- loss: 0.009244
0.4600 --- loss: 0.005527
0.4622 --- loss: 0.007364
0.4644 --- loss: 0.023021
0.4667 --- loss: 0.008214
0.4689 --- loss: 0.005295
0.4711 --- loss: 0.012324
0.4733 --- loss: 0.008777
0.4756 --- loss: 0.005752
0.4778 --- loss: 0.009739
0.4800 --- loss: 0.003932
0.4822 --- loss: 0.017565
0.4844 --- loss: 0.007211
0.4867 --- loss: 0.013972
0.4889 --- loss: 0.005168
0.4911 --- loss: 0.008166
0.4933 --- loss: 0.014024
0.4956 --- loss: 0.005821
0.4978 --- loss: 0.009251
0.5000 --- loss: 0.007284
0.5022 --- loss: 0.005654
0.5044 --- loss: 0.005586
0.5067 --- loss: 0.009715
0.5089 --- loss: 0.005929
0.5111 --- loss: 0.013050
0.5133 --- loss: 0.008993
0.5156 --- loss: 0.005500
0.5178 --- loss: 0.005882
0.5200 --- loss: 0.005801
0.5222 --- loss: 0.008362
0.5244 --- loss: 0.007459
0.5267 --- loss: 0.006423
0.5289 --- loss: 0.003985
0.5311 --- loss: 0.008916
0.5333 --- loss: 0.005287
0.5356 --- loss: 0.011183
0.5378 --- loss: 0.007468
0.5400 --- loss: 0.010915
0.5422 --- loss: 0.005620
0.5444 --- loss: 0.004796
0.5467 --- loss: 0.007842
0.5489 --- loss: 0.006797
0.5511 --- loss: 0.005580
0.5533 --- loss: 0.008316
0.5556 --- loss: 0.004227
0.5578 --- loss: 0.011071
0.5600 --- loss: 0.007432
0.5622 --- loss: 0.008619
0.5644 --- loss: 0.008837
0.5667 --- loss: 0.008129
0.5689 --- loss: 0.006339
0.5711 --- loss: 0.006293
0.5733 --- loss: 0.010519
0.5756 --- loss: 0.004236
0.5778 --- loss: 0.010523
0.5800 --- loss: 0.006433
0.5822 --- loss: 0.007955
0.5844 --- loss: 0.012116
0.5867 --- loss: 0.003519
0.5889 --- loss: 0.007691
0.5911 --- loss: 0.007573
0.5933 --- loss: 0.006899
0.5956 --- loss: 0.009652
0.5978 --- loss: 0.009066
0.6000 --- loss: 0.009220
0.6022 --- loss: 0.004405
0.6044 --- loss: 0.008611
0.6067 --- loss: 0.005767
0.6089 --- loss: 0.009308
0.6111 --- loss: 0.007994
0.6133 --- loss: 0.008493
0.6156 --- loss: 0.006254
0.6178 --- loss: 0.009233
0.6200 --- loss: 0.004891
0.6222 --- loss: 0.008363
0.6244 --- loss: 0.007048
0.6267 --- loss: 0.008032
0.6289 --- loss: 0.007369
0.6311 --- loss: 0.008475
0.6333 --- loss: 0.008919
0.6356 --- loss: 0.007321
0.6378 --- loss: 0.005408
0.6400 --- loss: 0.010030
0.6422 --- loss: 0.010683
0.6444 --- loss: 0.004904
0.6467 --- loss: 0.008161
0.6489 --- loss: 0.009062
0.6511 --- loss: 0.006881
0.6533 --- loss: 0.012307
0.6556 --- loss: 0.008707
0.6578 --- loss: 0.007954
0.6600 --- loss: 0.005138
0.6622 --- loss: 0.007183
0.6644 --- loss: 0.009401
0.6667 --- loss: 0.009578
0.6689 --- loss: 0.006833
0.6711 --- loss: 0.004768
0.6733 --- loss: 0.008121
0.6756 --- loss: 0.013452
0.6778 --- loss: 0.004181
0.6800 --- loss: 0.007431
0.6822 --- loss: 0.010672
0.6844 --- loss: 0.012341
0.6867 --- loss: 0.007271
0.6889 --- loss: 0.006590
0.6911 --- loss: 0.012174
0.6933 --- loss: 0.017240
0.6956 --- loss: 0.008637
0.6978 --- loss: 0.009958
0.7000 --- loss: 0.005871
0.7022 --- loss: 0.009341
0.7044 --- loss: 0.007623
0.7067 --- loss: 0.005754
0.7089 --- loss: 0.008120
0.7111 --- loss: 0.008096
0.7133 --- loss: 0.008715
0.7156 --- loss: 0.007275
0.7178 --- loss: 0.003989
0.7200 --- loss: 0.004940
0.7222 --- loss: 0.010038
0.7244 --- loss: 0.005871
0.7267 --- loss: 0.012105
0.7289 --- loss: 0.007406
0.7311 --- loss: 0.006569
0.7333 --- loss: 0.004997
0.7356 --- loss: 0.011128
0.7378 --- loss: 0.008950
0.7400 --- loss: 0.008831
0.7422 --- loss: 0.007168
0.7444 --- loss: 0.011351
0.7467 --- loss: 0.005369
0.7489 --- loss: 0.007702
0.7511 --- loss: 0.012509
0.7533 --- loss: 0.008817
0.7556 --- loss: 0.009482
0.7578 --- loss: 0.009365
0.7600 --- loss: 0.006898
0.7622 --- loss: 0.009807
0.7644 --- loss: 0.011179
0.7667 --- loss: 0.005343
0.7689 --- loss: 0.035384
0.7711 --- loss: 0.006910
0.7733 --- loss: 0.005232
0.7756 --- loss: 0.007143
0.7778 --- loss: 0.006508
0.7800 --- loss: 0.006616
0.7822 --- loss: 0.006321
0.7844 --- loss: 0.006969
0.7867 --- loss: 0.012106
0.7889 --- loss: 0.008029
0.7911 --- loss: 0.008480
0.7933 --- loss: 0.005406
0.7956 --- loss: 0.008068
0.7978 --- loss: 0.010516
0.8000 --- loss: 0.004774
0.8022 --- loss: 0.005889
0.8044 --- loss: 0.008378
0.8067 --- loss: 0.011026
0.8089 --- loss: 0.007923
0.8111 --- loss: 0.007729
0.8133 --- loss: 0.007588
0.8156 --- loss: 0.005514
0.8178 --- loss: 0.019870
0.8200 --- loss: 0.010771
0.8222 --- loss: 0.008421
0.8244 --- loss: 0.008084
0.8267 --- loss: 0.009043
0.8289 --- loss: 0.005785
0.8311 --- loss: 0.006910
0.8333 --- loss: 0.006116
0.8356 --- loss: 0.011939
0.8378 --- loss: 0.007116
0.8400 --- loss: 0.009278
0.8422 --- loss: 0.006392
0.8444 --- loss: 0.009524
0.8467 --- loss: 0.009438
0.8489 --- loss: 0.016838
0.8511 --- loss: 0.007643
0.8533 --- loss: 0.013089
0.8556 --- loss: 0.006840
0.8578 --- loss: 0.007557
0.8600 --- loss: 0.012491
0.8622 --- loss: 0.007116
0.8644 --- loss: 0.003906
0.8667 --- loss: 0.013259
0.8689 --- loss: 0.004705
0.8711 --- loss: 0.005995
0.8733 --- loss: 0.005444
0.8756 --- loss: 0.017274
0.8778 --- loss: 0.003958
0.8800 --- loss: 0.006281
0.8822 --- loss: 0.004953
0.8844 --- loss: 0.006979
0.8867 --- loss: 0.010214
0.8889 --- loss: 0.010029
0.8911 --- loss: 0.006161
0.8933 --- loss: 0.005739
0.8956 --- loss: 0.009579
0.8978 --- loss: 0.005393
0.9000 --- loss: 0.006178
0.9022 --- loss: 0.006060
0.9044 --- loss: 0.011769
0.9067 --- loss: 0.004996
0.9089 --- loss: 0.008482
0.9111 --- loss: 0.006601
0.9133 --- loss: 0.006761
0.9156 --- loss: 0.004506
0.9178 --- loss: 0.007964
0.9200 --- loss: 0.009574
0.9222 --- loss: 0.009414
0.9244 --- loss: 0.013217
0.9267 --- loss: 0.007418
0.9289 --- loss: 0.011967
0.9311 --- loss: 0.005896
0.9333 --- loss: 0.007414
0.9356 --- loss: 0.006498
0.9378 --- loss: 0.007070
0.9400 --- loss: 0.012513
0.9422 --- loss: 0.009986
0.9444 --- loss: 0.011152
0.9467 --- loss: 0.007513
0.9489 --- loss: 0.006656
0.9511 --- loss: 0.006594
0.9533 --- loss: 0.005040
0.9556 --- loss: 0.011037
0.9578 --- loss: 0.006250
0.9600 --- loss: 0.010107
0.9622 --- loss: 0.006311
0.9644 --- loss: 0.006278
0.9667 --- loss: 0.006997
0.9689 --- loss: 0.008151
0.9711 --- loss: 0.007918
0.9733 --- loss: 0.010628
0.9756 --- loss: 0.008154
0.9778 --- loss: 0.007001
0.9800 --- loss: 0.012643
0.9822 --- loss: 0.005738
0.9844 --- loss: 0.007241
0.9867 --- loss: 0.007637
0.9889 --- loss: 0.006841
0.9911 --- loss: 0.020350
0.9933 --- loss: 0.006455
0.9956 --- loss: 0.008019
0.9978 --- loss: 0.006940
Epoch finished ! Loss: 0.00915089731347355
Checkpoint e3 saved !

real	5m31.886s
user	34m27.837s
sys	1m51.235s
