Model loaded from mp-l23-IonAndScint-cosmic/CP1-450.pth
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449]

    Starting training:
        Epochs: 2
        Batch size: 1
        Learning rate: 0.1
        Training size: 450
        Validation size: 50
        Checkpoints: True
        CUDA: True
    

    im_tags: ['frame_loose_lf0', 'frame_mp2_roi0', 'frame_mp3_roi0']
    ma_tags: ['frame_ductor0']
    truth_th: 100
    

        file_img: data/mu-rec-0.h5
        file_mask: data/mu-tru-0.h5
        
Starting epoch 2/2.
0.0000 --- loss: 0.011531
0.0022 --- loss: 0.008710
0.0044 --- loss: 0.005437
0.0067 --- loss: 0.004085
0.0089 --- loss: 0.005274
0.0111 --- loss: 0.003060
0.0133 --- loss: 0.002537
0.0156 --- loss: 0.002291
0.0178 --- loss: 0.002961
0.0200 --- loss: 0.003839
0.0222 --- loss: 0.002125
0.0244 --- loss: 0.001645
0.0267 --- loss: 0.001926
0.0289 --- loss: 0.001196
0.0311 --- loss: 0.001573
0.0333 --- loss: 0.002071
0.0356 --- loss: 0.001209
0.0378 --- loss: 0.001675
0.0400 --- loss: 0.001743
0.0422 --- loss: 0.002118
0.0444 --- loss: 0.001026
0.0467 --- loss: 0.001164
0.0489 --- loss: 0.002064
0.0511 --- loss: 0.001410
0.0533 --- loss: 0.001525
0.0556 --- loss: 0.001441
0.0578 --- loss: 0.001480
0.0600 --- loss: 0.000780
0.0622 --- loss: 0.001487
0.0644 --- loss: 0.001543
0.0667 --- loss: 0.001704
0.0689 --- loss: 0.001379
0.0711 --- loss: 0.000761
0.0733 --- loss: 0.001235
0.0756 --- loss: 0.001392
0.0778 --- loss: 0.000935
0.0800 --- loss: 0.001272
0.0822 --- loss: 0.001720
0.0844 --- loss: 0.002092
0.0867 --- loss: 0.001697
0.0889 --- loss: 0.001411
0.0911 --- loss: 0.001783
0.0933 --- loss: 0.001111
0.0956 --- loss: 0.000792
0.0978 --- loss: 0.000775
0.1000 --- loss: 0.001690
0.1022 --- loss: 0.001361
0.1044 --- loss: 0.000938
0.1067 --- loss: 0.000629
0.1089 --- loss: 0.001074
0.1111 --- loss: 0.001522
0.1133 --- loss: 0.000842
0.1156 --- loss: 0.001063
0.1178 --- loss: 0.001080
0.1200 --- loss: 0.001258
0.1222 --- loss: 0.000611
0.1244 --- loss: 0.002474
0.1267 --- loss: 0.001133
0.1289 --- loss: 0.001219
0.1311 --- loss: 0.001551
0.1333 --- loss: 0.001149
0.1356 --- loss: 0.001212
0.1378 --- loss: 0.001534
0.1400 --- loss: 0.001287
0.1422 --- loss: 0.001595
0.1444 --- loss: 0.000841
0.1467 --- loss: 0.001076
0.1489 --- loss: 0.000665
0.1511 --- loss: 0.001131
0.1533 --- loss: 0.001640
0.1556 --- loss: 0.000733
0.1578 --- loss: 0.001276
0.1600 --- loss: 0.001326
0.1622 --- loss: 0.001061
0.1644 --- loss: 0.000977
0.1667 --- loss: 0.000755
0.1689 --- loss: 0.001170
0.1711 --- loss: 0.000798
0.1733 --- loss: 0.000896
0.1756 --- loss: 0.000711
0.1778 --- loss: 0.000935
0.1800 --- loss: 0.001432
0.1822 --- loss: 0.001338
0.1844 --- loss: 0.002160
0.1867 --- loss: 0.001927
0.1889 --- loss: 0.000904
0.1911 --- loss: 0.001023
0.1933 --- loss: 0.002203
0.1956 --- loss: 0.000822
0.1978 --- loss: 0.000897
0.2000 --- loss: 0.000857
0.2022 --- loss: 0.001457
0.2044 --- loss: 0.000891
0.2067 --- loss: 0.001012
0.2089 --- loss: 0.001628
0.2111 --- loss: 0.001277
0.2133 --- loss: 0.001573
0.2156 --- loss: 0.000680
0.2178 --- loss: 0.001376
0.2200 --- loss: 0.001404
0.2222 --- loss: 0.001006
0.2244 --- loss: 0.000859
0.2267 --- loss: 0.000712
0.2289 --- loss: 0.000695
0.2311 --- loss: 0.001062
0.2333 --- loss: 0.000711
0.2356 --- loss: 0.000589
0.2378 --- loss: 0.001379
0.2400 --- loss: 0.001250
0.2422 --- loss: 0.000921
0.2444 --- loss: 0.001014
0.2467 --- loss: 0.001607
0.2489 --- loss: 0.001150
0.2511 --- loss: 0.000786
0.2533 --- loss: 0.000740
0.2556 --- loss: 0.001050
0.2578 --- loss: 0.001476
0.2600 --- loss: 0.000853
0.2622 --- loss: 0.001313
0.2644 --- loss: 0.001247
0.2667 --- loss: 0.001283
0.2689 --- loss: 0.000819
0.2711 --- loss: 0.001655
0.2733 --- loss: 0.001253
0.2756 --- loss: 0.000671
0.2778 --- loss: 0.000789
0.2800 --- loss: 0.001953
0.2822 --- loss: 0.000874
0.2844 --- loss: 0.001258
0.2867 --- loss: 0.001600
0.2889 --- loss: 0.000817
0.2911 --- loss: 0.001269
0.2933 --- loss: 0.001074
0.2956 --- loss: 0.001008
0.2978 --- loss: 0.000800
0.3000 --- loss: 0.000739
0.3022 --- loss: 0.000760
0.3044 --- loss: 0.001174
0.3067 --- loss: 0.001872
0.3089 --- loss: 0.001198
0.3111 --- loss: 0.000621
0.3133 --- loss: 0.000863
0.3156 --- loss: 0.000827
0.3178 --- loss: 0.000670
0.3200 --- loss: 0.000909
0.3222 --- loss: 0.000875
0.3244 --- loss: 0.001362
0.3267 --- loss: 0.001406
0.3289 --- loss: 0.000945
0.3311 --- loss: 0.001109
0.3333 --- loss: 0.000848
0.3356 --- loss: 0.001240
0.3378 --- loss: 0.000940
0.3400 --- loss: 0.000693
0.3422 --- loss: 0.000730
0.3444 --- loss: 0.000938
0.3467 --- loss: 0.001223
0.3489 --- loss: 0.000611
0.3511 --- loss: 0.000608
0.3533 --- loss: 0.001185
0.3556 --- loss: 0.000869
0.3578 --- loss: 0.001576
0.3600 --- loss: 0.001585
0.3622 --- loss: 0.000746
0.3644 --- loss: 0.000840
0.3667 --- loss: 0.001548
0.3689 --- loss: 0.000777
0.3711 --- loss: 0.001000
0.3733 --- loss: 0.000998
0.3756 --- loss: 0.000814
0.3778 --- loss: 0.001327
0.3800 --- loss: 0.000833
0.3822 --- loss: 0.000753
0.3844 --- loss: 0.000776
0.3867 --- loss: 0.000840
0.3889 --- loss: 0.000566
0.3911 --- loss: 0.000914
0.3933 --- loss: 0.000586
0.3956 --- loss: 0.000616
0.3978 --- loss: 0.000573
0.4000 --- loss: 0.001007
0.4022 --- loss: 0.000591
0.4044 --- loss: 0.001274
0.4067 --- loss: 0.001208
0.4089 --- loss: 0.000747
0.4111 --- loss: 0.000928
0.4133 --- loss: 0.000848
0.4156 --- loss: 0.000999
0.4178 --- loss: 0.001434
0.4200 --- loss: 0.000912
0.4222 --- loss: 0.000958
0.4244 --- loss: 0.000720
0.4267 --- loss: 0.002173
0.4289 --- loss: 0.001218
0.4311 --- loss: 0.000662
0.4333 --- loss: 0.001059
0.4356 --- loss: 0.000671
0.4378 --- loss: 0.000779
0.4400 --- loss: 0.000828
0.4422 --- loss: 0.001420
0.4444 --- loss: 0.001408
0.4467 --- loss: 0.000667
0.4489 --- loss: 0.001078
0.4511 --- loss: 0.000940
0.4533 --- loss: 0.001425
0.4556 --- loss: 0.000931
0.4578 --- loss: 0.000849
0.4600 --- loss: 0.000880
0.4622 --- loss: 0.000746
0.4644 --- loss: 0.001578
0.4667 --- loss: 0.001031
0.4689 --- loss: 0.000799
0.4711 --- loss: 0.001513
0.4733 --- loss: 0.001102
0.4756 --- loss: 0.001015
0.4778 --- loss: 0.001198
0.4800 --- loss: 0.000689
0.4822 --- loss: 0.000912
0.4844 --- loss: 0.000735
0.4867 --- loss: 0.000588
0.4889 --- loss: 0.000812
0.4911 --- loss: 0.000828
0.4933 --- loss: 0.000850
0.4956 --- loss: 0.001395
0.4978 --- loss: 0.001235
0.5000 --- loss: 0.001619
0.5022 --- loss: 0.001026
0.5044 --- loss: 0.000796
0.5067 --- loss: 0.001043
0.5089 --- loss: 0.001000
0.5111 --- loss: 0.000844
0.5133 --- loss: 0.001159
0.5156 --- loss: 0.001125
0.5178 --- loss: 0.001483
0.5200 --- loss: 0.001281
0.5222 --- loss: 0.001376
0.5244 --- loss: 0.000874
0.5267 --- loss: 0.001604
0.5289 --- loss: 0.001250
0.5311 --- loss: 0.000962
0.5333 --- loss: 0.000868
0.5356 --- loss: 0.001129
0.5378 --- loss: 0.001650
0.5400 --- loss: 0.001180
0.5422 --- loss: 0.001097
0.5444 --- loss: 0.001390
0.5467 --- loss: 0.001721
0.5489 --- loss: 0.000729
0.5511 --- loss: 0.000881
0.5533 --- loss: 0.000629
0.5556 --- loss: 0.001191
0.5578 --- loss: 0.000695
0.5600 --- loss: 0.000992
0.5622 --- loss: 0.001024
0.5644 --- loss: 0.001138
0.5667 --- loss: 0.001420
0.5689 --- loss: 0.001377
0.5711 --- loss: 0.001319
0.5733 --- loss: 0.000856
0.5756 --- loss: 0.000817
0.5778 --- loss: 0.001377
0.5800 --- loss: 0.001507
0.5822 --- loss: 0.001073
0.5844 --- loss: 0.001079
0.5867 --- loss: 0.001416
0.5889 --- loss: 0.001037
0.5911 --- loss: 0.001104
0.5933 --- loss: 0.000879
0.5956 --- loss: 0.000612
0.5978 --- loss: 0.001066
0.6000 --- loss: 0.000687
0.6022 --- loss: 0.001492
0.6044 --- loss: 0.001209
0.6067 --- loss: 0.000945
0.6089 --- loss: 0.000822
0.6111 --- loss: 0.000795
0.6133 --- loss: 0.001714
0.6156 --- loss: 0.001071
0.6178 --- loss: 0.000872
0.6200 --- loss: 0.001030
0.6222 --- loss: 0.000707
0.6244 --- loss: 0.001293
0.6267 --- loss: 0.001670
0.6289 --- loss: 0.000889
0.6311 --- loss: 0.000899
0.6333 --- loss: 0.000749
0.6356 --- loss: 0.000680
0.6378 --- loss: 0.002270
0.6400 --- loss: 0.001280
0.6422 --- loss: 0.000617
0.6444 --- loss: 0.000858
0.6467 --- loss: 0.001441
0.6489 --- loss: 0.001069
0.6511 --- loss: 0.001521
0.6533 --- loss: 0.000963
0.6556 --- loss: 0.001297
0.6578 --- loss: 0.001360
0.6600 --- loss: 0.001207
0.6622 --- loss: 0.001573
0.6644 --- loss: 0.001233
0.6667 --- loss: 0.000709
0.6689 --- loss: 0.001241
0.6711 --- loss: 0.001365
0.6733 --- loss: 0.000622
0.6756 --- loss: 0.000840
0.6778 --- loss: 0.000910
0.6800 --- loss: 0.001372
0.6822 --- loss: 0.001749
0.6844 --- loss: 0.001171
0.6867 --- loss: 0.001185
0.6889 --- loss: 0.001168
0.6911 --- loss: 0.000933
0.6933 --- loss: 0.000846
0.6956 --- loss: 0.000724
0.6978 --- loss: 0.001382
0.7000 --- loss: 0.000848
0.7022 --- loss: 0.000869
0.7044 --- loss: 0.001621
0.7067 --- loss: 0.001077
0.7089 --- loss: 0.001237
0.7111 --- loss: 0.001492
0.7133 --- loss: 0.001407
0.7156 --- loss: 0.001560
0.7178 --- loss: 0.001070
0.7200 --- loss: 0.000967
0.7222 --- loss: 0.001132
0.7244 --- loss: 0.000933
0.7267 --- loss: 0.001313
0.7289 --- loss: 0.001211
0.7311 --- loss: 0.000974
0.7333 --- loss: 0.000796
0.7356 --- loss: 0.001079
0.7378 --- loss: 0.000820
0.7400 --- loss: 0.001195
0.7422 --- loss: 0.001294
0.7444 --- loss: 0.001065
0.7467 --- loss: 0.001015
0.7489 --- loss: 0.000877
0.7511 --- loss: 0.001564
0.7533 --- loss: 0.001026
0.7556 --- loss: 0.001300
0.7578 --- loss: 0.001484
0.7600 --- loss: 0.001173
0.7622 --- loss: 0.000678
0.7644 --- loss: 0.000989
0.7667 --- loss: 0.001005
0.7689 --- loss: 0.000784
0.7711 --- loss: 0.001089
0.7733 --- loss: 0.001437
0.7756 --- loss: 0.001280
0.7778 --- loss: 0.000868
0.7800 --- loss: 0.001549
0.7822 --- loss: 0.001630
0.7844 --- loss: 0.000875
0.7867 --- loss: 0.000907
0.7889 --- loss: 0.001253
0.7911 --- loss: 0.001387
0.7933 --- loss: 0.000824
0.7956 --- loss: 0.001120
0.7978 --- loss: 0.001135
0.8000 --- loss: 0.000895
0.8022 --- loss: 0.000748
0.8044 --- loss: 0.000755
0.8067 --- loss: 0.001468
0.8089 --- loss: 0.001191
0.8111 --- loss: 0.001249
0.8133 --- loss: 0.001810
0.8156 --- loss: 0.001266
0.8178 --- loss: 0.000840
0.8200 --- loss: 0.001151
0.8222 --- loss: 0.001359
0.8244 --- loss: 0.001412
0.8267 --- loss: 0.000692
0.8289 --- loss: 0.000719
0.8311 --- loss: 0.002150
0.8333 --- loss: 0.000886
0.8356 --- loss: 0.001442
0.8378 --- loss: 0.000846
0.8400 --- loss: 0.000681
0.8422 --- loss: 0.001465
0.8444 --- loss: 0.001230
0.8467 --- loss: 0.001139
0.8489 --- loss: 0.001225
0.8511 --- loss: 0.001396
0.8533 --- loss: 0.001842
0.8556 --- loss: 0.000790
0.8578 --- loss: 0.001764
0.8600 --- loss: 0.001082
0.8622 --- loss: 0.001280
0.8644 --- loss: 0.001099
0.8667 --- loss: 0.001287
0.8689 --- loss: 0.001192
0.8711 --- loss: 0.001566
0.8733 --- loss: 0.001432
0.8756 --- loss: 0.000901
0.8778 --- loss: 0.000851
0.8800 --- loss: 0.000969
0.8822 --- loss: 0.000755
0.8844 --- loss: 0.000927
0.8867 --- loss: 0.000713
0.8889 --- loss: 0.001228
0.8911 --- loss: 0.001202
0.8933 --- loss: 0.000810
0.8956 --- loss: 0.001117
0.8978 --- loss: 0.000941
0.9000 --- loss: 0.001026
0.9022 --- loss: 0.001442
0.9044 --- loss: 0.002372
0.9067 --- loss: 0.000697
0.9089 --- loss: 0.001258
0.9111 --- loss: 0.001172
0.9133 --- loss: 0.000751
0.9156 --- loss: 0.001532
0.9178 --- loss: 0.000710
0.9200 --- loss: 0.001850
0.9222 --- loss: 0.001055
0.9244 --- loss: 0.002179
0.9267 --- loss: 0.001022
0.9289 --- loss: 0.001537
0.9311 --- loss: 0.001049
0.9333 --- loss: 0.001149
0.9356 --- loss: 0.000919
0.9378 --- loss: 0.000984
0.9400 --- loss: 0.001731
0.9422 --- loss: 0.001202
0.9444 --- loss: 0.000904
0.9467 --- loss: 0.001334
0.9489 --- loss: 0.001272
0.9511 --- loss: 0.000877
0.9533 --- loss: 0.000795
0.9556 --- loss: 0.000885
0.9578 --- loss: 0.001459
0.9600 --- loss: 0.000922
0.9622 --- loss: 0.001383
0.9644 --- loss: 0.000942
0.9667 --- loss: 0.001501
0.9689 --- loss: 0.001077
0.9711 --- loss: 0.001023
0.9733 --- loss: 0.000972
0.9756 --- loss: 0.000949
0.9778 --- loss: 0.001091
0.9800 --- loss: 0.000978
0.9822 --- loss: 0.001079
0.9844 --- loss: 0.000838
0.9867 --- loss: 0.000814
0.9889 --- loss: 0.001070
0.9911 --- loss: 0.001102
0.9933 --- loss: 0.000856
Epoch finished ! Loss: 0.0012236528777802818
Checkpoint e2 saved !

real	5m23.770s
user	32m52.651s
sys	1m44.084s
