Model loaded from checkpoints/CP3-450.pth
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449]

    Starting training:
        Epochs: 4
        Batch size: 1
        Learning rate: 0.1
        Training size: 450
        Validation size: 50
        Checkpoints: True
        CUDA: True
    

    im_tags: ['frame_loose_lf0', 'frame_mp2_roi0', 'frame_mp3_roi0']
    ma_tags: ['frame_ductor0']
    truth_th: 100
    

        file_img: data/mu-rec-0.h5
        file_mask: data/mu-tru-0.h5
        
Starting epoch 4/4.
0.0000 --- loss: 0.007261
0.0022 --- loss: 0.005313
0.0044 --- loss: 0.004950
0.0067 --- loss: 0.004112
0.0089 --- loss: 0.004643
0.0111 --- loss: 0.002585
0.0133 --- loss: 0.002369
0.0156 --- loss: 0.002394
0.0178 --- loss: 0.002985
0.0200 --- loss: 0.003360
0.0222 --- loss: 0.002120
0.0244 --- loss: 0.001853
0.0267 --- loss: 0.002078
0.0289 --- loss: 0.001835
0.0311 --- loss: 0.002226
0.0333 --- loss: 0.002583
0.0356 --- loss: 0.001558
0.0378 --- loss: 0.001957
0.0400 --- loss: 0.002068
0.0422 --- loss: 0.002336
0.0444 --- loss: 0.001424
0.0467 --- loss: 0.001650
0.0489 --- loss: 0.002609
0.0511 --- loss: 0.002005
0.0533 --- loss: 0.001966
0.0556 --- loss: 0.001890
0.0578 --- loss: 0.001923
0.0600 --- loss: 0.001267
0.0622 --- loss: 0.001907
0.0644 --- loss: 0.001863
0.0667 --- loss: 0.001861
0.0689 --- loss: 0.001675
0.0711 --- loss: 0.001295
0.0733 --- loss: 0.001919
0.0756 --- loss: 0.001807
0.0778 --- loss: 0.001349
0.0800 --- loss: 0.001685
0.0822 --- loss: 0.001296
0.0844 --- loss: 0.002129
0.0867 --- loss: 0.002015
0.0889 --- loss: 0.001803
0.0911 --- loss: 0.002274
0.0933 --- loss: 0.001368
0.0956 --- loss: 0.001108
0.0978 --- loss: 0.001150
0.1000 --- loss: 0.002462
0.1022 --- loss: 0.001761
0.1044 --- loss: 0.001433
0.1067 --- loss: 0.001056
0.1089 --- loss: 0.001479
0.1111 --- loss: 0.001891
0.1133 --- loss: 0.001253
0.1156 --- loss: 0.001478
0.1178 --- loss: 0.001328
0.1200 --- loss: 0.001584
0.1222 --- loss: 0.001072
0.1244 --- loss: 0.002773
0.1267 --- loss: 0.001476
0.1289 --- loss: 0.001473
0.1311 --- loss: 0.002004
0.1333 --- loss: 0.001405
0.1356 --- loss: 0.001497
0.1378 --- loss: 0.001850
0.1400 --- loss: 0.001591
0.1422 --- loss: 0.001837
0.1444 --- loss: 0.001225
0.1467 --- loss: 0.001375
0.1489 --- loss: 0.001010
0.1511 --- loss: 0.001473
0.1533 --- loss: 0.001881
0.1556 --- loss: 0.001101
0.1578 --- loss: 0.001691
0.1600 --- loss: 0.001616
0.1622 --- loss: 0.001458
0.1644 --- loss: 0.001287
0.1667 --- loss: 0.001066
0.1689 --- loss: 0.001392
0.1711 --- loss: 0.001122
0.1733 --- loss: 0.001249
0.1756 --- loss: 0.001032
0.1778 --- loss: 0.001288
0.1800 --- loss: 0.001668
0.1822 --- loss: 0.001595
0.1844 --- loss: 0.002246
0.1867 --- loss: 0.002199
0.1889 --- loss: 0.001186
0.1911 --- loss: 0.001332
0.1933 --- loss: 0.002328
0.1956 --- loss: 0.001137
0.1978 --- loss: 0.001226
0.2000 --- loss: 0.001187
0.2022 --- loss: 0.001729
0.2044 --- loss: 0.001176
0.2067 --- loss: 0.001446
0.2089 --- loss: 0.001842
0.2111 --- loss: 0.001568
0.2133 --- loss: 0.001717
0.2156 --- loss: 0.001017
0.2178 --- loss: 0.001735
0.2200 --- loss: 0.001603
0.2222 --- loss: 0.001279
0.2244 --- loss: 0.001152
0.2267 --- loss: 0.000989
0.2289 --- loss: 0.000997
0.2311 --- loss: 0.001340
0.2333 --- loss: 0.001000
0.2356 --- loss: 0.000904
0.2378 --- loss: 0.001688
0.2400 --- loss: 0.001462
0.2422 --- loss: 0.001366
0.2444 --- loss: 0.001285
0.2467 --- loss: 0.001824
0.2489 --- loss: 0.001436
0.2511 --- loss: 0.001007
0.2533 --- loss: 0.001044
0.2556 --- loss: 0.001353
0.2578 --- loss: 0.001881
0.2600 --- loss: 0.001242
0.2622 --- loss: 0.001519
0.2644 --- loss: 0.001388
0.2667 --- loss: 0.001510
0.2689 --- loss: 0.001109
0.2711 --- loss: 0.001924
0.2733 --- loss: 0.001579
0.2756 --- loss: 0.000983
0.2778 --- loss: 0.000913
0.2800 --- loss: 0.002047
0.2822 --- loss: 0.001036
0.2844 --- loss: 0.001416
0.2867 --- loss: 0.001841
0.2889 --- loss: 0.001079
0.2911 --- loss: 0.001484
0.2933 --- loss: 0.001267
0.2956 --- loss: 0.001228
0.2978 --- loss: 0.000985
0.3000 --- loss: 0.001055
0.3022 --- loss: 0.001034
0.3044 --- loss: 0.001444
0.3067 --- loss: 0.002040
0.3089 --- loss: 0.001408
0.3111 --- loss: 0.000883
0.3133 --- loss: 0.001136
0.3156 --- loss: 0.001133
0.3178 --- loss: 0.000935
0.3200 --- loss: 0.001144
0.3222 --- loss: 0.001165
0.3244 --- loss: 0.001602
0.3267 --- loss: 0.001620
0.3289 --- loss: 0.001148
0.3311 --- loss: 0.001307
0.3333 --- loss: 0.001097
0.3356 --- loss: 0.001434
0.3378 --- loss: 0.001169
0.3400 --- loss: 0.000999
0.3422 --- loss: 0.000962
0.3444 --- loss: 0.001113
0.3467 --- loss: 0.001500
0.3489 --- loss: 0.000856
0.3511 --- loss: 0.000870
0.3533 --- loss: 0.001282
0.3556 --- loss: 0.001116
0.3578 --- loss: 0.001634
0.3600 --- loss: 0.001671
0.3622 --- loss: 0.001003
0.3644 --- loss: 0.001063
0.3667 --- loss: 0.001631
0.3689 --- loss: 0.001064
0.3711 --- loss: 0.001271
0.3733 --- loss: 0.001138
0.3756 --- loss: 0.000993
0.3778 --- loss: 0.001451
0.3800 --- loss: 0.001067
0.3822 --- loss: 0.000990
0.3844 --- loss: 0.001018
0.3867 --- loss: 0.001073
0.3889 --- loss: 0.000796
0.3911 --- loss: 0.001102
0.3933 --- loss: 0.000817
0.3956 --- loss: 0.000857
0.3978 --- loss: 0.000841
0.4000 --- loss: 0.001187
0.4022 --- loss: 0.000822
0.4044 --- loss: 0.001446
0.4067 --- loss: 0.001323
0.4089 --- loss: 0.000963
0.4111 --- loss: 0.001160
0.4133 --- loss: 0.001078
0.4156 --- loss: 0.001187
0.4178 --- loss: 0.001557
0.4200 --- loss: 0.001105
0.4222 --- loss: 0.001162
0.4244 --- loss: 0.000913
0.4267 --- loss: 0.002218
0.4289 --- loss: 0.001467
0.4311 --- loss: 0.000912
0.4333 --- loss: 0.001315
0.4356 --- loss: 0.000918
0.4378 --- loss: 0.001004
0.4400 --- loss: 0.001004
0.4422 --- loss: 0.001600
0.4444 --- loss: 0.001456
0.4467 --- loss: 0.000895
0.4489 --- loss: 0.001307
0.4511 --- loss: 0.001107
0.4533 --- loss: 0.001679
0.4556 --- loss: 0.000885
0.4578 --- loss: 0.001031
0.4600 --- loss: 0.001069
0.4622 --- loss: 0.000910
0.4644 --- loss: 0.001674
0.4667 --- loss: 0.001198
0.4689 --- loss: 0.001005
0.4711 --- loss: 0.001720
0.4733 --- loss: 0.001191
0.4756 --- loss: 0.001121
0.4778 --- loss: 0.001402
0.4800 --- loss: 0.000891
0.4822 --- loss: 0.001108
0.4844 --- loss: 0.000960
0.4867 --- loss: 0.000808
0.4889 --- loss: 0.001074
0.4911 --- loss: 0.000987
0.4933 --- loss: 0.001063
0.4956 --- loss: 0.001501
0.4978 --- loss: 0.001415
0.5000 --- loss: 0.001777
0.5022 --- loss: 0.001230
0.5044 --- loss: 0.001017
0.5067 --- loss: 0.001171
0.5089 --- loss: 0.001071
0.5111 --- loss: 0.001000
0.5133 --- loss: 0.001334
0.5156 --- loss: 0.001311
0.5178 --- loss: 0.001635
0.5200 --- loss: 0.001394
0.5222 --- loss: 0.001537
0.5244 --- loss: 0.001013
0.5267 --- loss: 0.001731
0.5289 --- loss: 0.001358
0.5311 --- loss: 0.001134
0.5333 --- loss: 0.001003
0.5356 --- loss: 0.001278
0.5378 --- loss: 0.001790
0.5400 --- loss: 0.001417
0.5422 --- loss: 0.001194
0.5444 --- loss: 0.001468
0.5467 --- loss: 0.001834
0.5489 --- loss: 0.000895
0.5511 --- loss: 0.001032
0.5533 --- loss: 0.000828
0.5556 --- loss: 0.001380
0.5578 --- loss: 0.000859
0.5600 --- loss: 0.001256
0.5622 --- loss: 0.000836
0.5644 --- loss: 0.001077
0.5667 --- loss: 0.001323
0.5689 --- loss: 0.001432
0.5711 --- loss: 0.001422
0.5733 --- loss: 0.001060
0.5756 --- loss: 0.001049
0.5778 --- loss: 0.001351
0.5800 --- loss: 0.001502
0.5822 --- loss: 0.001184
0.5844 --- loss: 0.001189
0.5867 --- loss: 0.001507
0.5889 --- loss: 0.001081
0.5911 --- loss: 0.001147
0.5933 --- loss: 0.001038
0.5956 --- loss: 0.000784
0.5978 --- loss: 0.001207
0.6000 --- loss: 0.000856
0.6022 --- loss: 0.001577
0.6044 --- loss: 0.001349
0.6067 --- loss: 0.001105
0.6089 --- loss: 0.000983
0.6111 --- loss: 0.000965
0.6133 --- loss: 0.001953
0.6156 --- loss: 0.001222
0.6178 --- loss: 0.001014
0.6200 --- loss: 0.001204
0.6222 --- loss: 0.000902
0.6244 --- loss: 0.001395
0.6267 --- loss: 0.001828
0.6289 --- loss: 0.001014
0.6311 --- loss: 0.001062
0.6333 --- loss: 0.000905
0.6356 --- loss: 0.000882
0.6378 --- loss: 0.002204
0.6400 --- loss: 0.001371
0.6422 --- loss: 0.000808
0.6444 --- loss: 0.001026
0.6467 --- loss: 0.001584
0.6489 --- loss: 0.001267
0.6511 --- loss: 0.001626
0.6533 --- loss: 0.001089
0.6556 --- loss: 0.001338
0.6578 --- loss: 0.001506
0.6600 --- loss: 0.001281
0.6622 --- loss: 0.001618
0.6644 --- loss: 0.001337
0.6667 --- loss: 0.000833
0.6689 --- loss: 0.001381
0.6711 --- loss: 0.001466
0.6733 --- loss: 0.000786
0.6756 --- loss: 0.000956
0.6778 --- loss: 0.001043
0.6800 --- loss: 0.001483
0.6822 --- loss: 0.001824
0.6844 --- loss: 0.001318
0.6867 --- loss: 0.001282
0.6889 --- loss: 0.001173
0.6911 --- loss: 0.001063
0.6933 --- loss: 0.000974
0.6956 --- loss: 0.000877
0.6978 --- loss: 0.001453
0.7000 --- loss: 0.000983
0.7022 --- loss: 0.000974
0.7044 --- loss: 0.001609
0.7067 --- loss: 0.001218
0.7089 --- loss: 0.001315
0.7111 --- loss: 0.001551
0.7133 --- loss: 0.001517
0.7156 --- loss: 0.001540
0.7178 --- loss: 0.001224
0.7200 --- loss: 0.001087
0.7222 --- loss: 0.001157
0.7244 --- loss: 0.001040
0.7267 --- loss: 0.001345
0.7289 --- loss: 0.001272
0.7311 --- loss: 0.001051
0.7333 --- loss: 0.000953
0.7356 --- loss: 0.001218
0.7378 --- loss: 0.000966
0.7400 --- loss: 0.001219
0.7422 --- loss: 0.001311
0.7444 --- loss: 0.001112
0.7467 --- loss: 0.001111
0.7489 --- loss: 0.000967
0.7511 --- loss: 0.001672
0.7533 --- loss: 0.001127
0.7556 --- loss: 0.001426
0.7578 --- loss: 0.001538
0.7600 --- loss: 0.001270
0.7622 --- loss: 0.000839
0.7644 --- loss: 0.001117
0.7667 --- loss: 0.001090
0.7689 --- loss: 0.000937
0.7711 --- loss: 0.001153
0.7733 --- loss: 0.001518
0.7756 --- loss: 0.001413
0.7778 --- loss: 0.000986
0.7800 --- loss: 0.001570
0.7822 --- loss: 0.001740
0.7844 --- loss: 0.000979
0.7867 --- loss: 0.001076
0.7889 --- loss: 0.001333
0.7911 --- loss: 0.001525
0.7933 --- loss: 0.000945
0.7956 --- loss: 0.001201
0.7978 --- loss: 0.001168
0.8000 --- loss: 0.001070
0.8022 --- loss: 0.000875
0.8044 --- loss: 0.000905
0.8067 --- loss: 0.001564
0.8089 --- loss: 0.001330
0.8111 --- loss: 0.001307
0.8133 --- loss: 0.001830
0.8156 --- loss: 0.001368
0.8178 --- loss: 0.000992
0.8200 --- loss: 0.001226
0.8222 --- loss: 0.001427
0.8244 --- loss: 0.001447
0.8267 --- loss: 0.000825
0.8289 --- loss: 0.000885
0.8311 --- loss: 0.002146
0.8333 --- loss: 0.001023
0.8356 --- loss: 0.001407
0.8378 --- loss: 0.000938
0.8400 --- loss: 0.000813
0.8422 --- loss: 0.001603
0.8444 --- loss: 0.001441
0.8467 --- loss: 0.001342
0.8489 --- loss: 0.001349
0.8511 --- loss: 0.001461
0.8533 --- loss: 0.001729
0.8556 --- loss: 0.000920
0.8578 --- loss: 0.001818
0.8600 --- loss: 0.001284
0.8622 --- loss: 0.001372
0.8644 --- loss: 0.001234
0.8667 --- loss: 0.001387
0.8689 --- loss: 0.001249
0.8711 --- loss: 0.001431
0.8733 --- loss: 0.001432
0.8756 --- loss: 0.000978
0.8778 --- loss: 0.001020
0.8800 --- loss: 0.000994
0.8822 --- loss: 0.000982
0.8844 --- loss: 0.001018
0.8867 --- loss: 0.000800
0.8889 --- loss: 0.001274
0.8911 --- loss: 0.001264
0.8933 --- loss: 0.000887
0.8956 --- loss: 0.001208
0.8978 --- loss: 0.001170
0.9000 --- loss: 0.001083
0.9022 --- loss: 0.001241
0.9044 --- loss: 0.002188
0.9067 --- loss: 0.000821
0.9089 --- loss: 0.001303
0.9111 --- loss: 0.001225
0.9133 --- loss: 0.000823
0.9156 --- loss: 0.001394
0.9178 --- loss: 0.000832
0.9200 --- loss: 0.001841
0.9222 --- loss: 0.001094
0.9244 --- loss: 0.002114
0.9267 --- loss: 0.001057
0.9289 --- loss: 0.001557
0.9311 --- loss: 0.001091
0.9333 --- loss: 0.001200
0.9356 --- loss: 0.000976
0.9378 --- loss: 0.001043
0.9400 --- loss: 0.001643
0.9422 --- loss: 0.001258
0.9444 --- loss: 0.000994
0.9467 --- loss: 0.001383
0.9489 --- loss: 0.001234
0.9511 --- loss: 0.000904
0.9533 --- loss: 0.000881
0.9556 --- loss: 0.000967
0.9578 --- loss: 0.001454
0.9600 --- loss: 0.001015
0.9622 --- loss: 0.001394
0.9644 --- loss: 0.001046
0.9667 --- loss: 0.001508
0.9689 --- loss: 0.001109
0.9711 --- loss: 0.000997
0.9733 --- loss: 0.001004
0.9756 --- loss: 0.000967
0.9778 --- loss: 0.001138
0.9800 --- loss: 0.001072
0.9822 --- loss: 0.001148
0.9844 --- loss: 0.000934
0.9867 --- loss: 0.000870
0.9889 --- loss: 0.001124
0.9911 --- loss: 0.001176
0.9933 --- loss: 0.000934
Epoch finished ! Loss: 0.0013775260121794938
Checkpoint e4 saved !

real	5m25.232s
user	32m54.163s
sys	1m48.310s
