Model loaded from mp-l23-IonAndScint-cosmic/CP5-450.pth
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449]

    Starting training:
        Epochs: 1
        Batch size: 1
        Learning rate: 0.1
        Training size: 450
        Validation size: 50
        Checkpoints: True
        CUDA: True
    

    file_img: data/mu-rec-0.h5
    file_mask: data/mu-tru-0.h5
    im_tags: ['frame_loose_lf0', 'frame_mp2_roi0', 'frame_mp3_roi0']
    ma_tags: ['frame_ductor0']
    truth_th: 100
    
Starting epoch 1/1.
0.0000 --- loss: 0.008342
0.0022 --- loss: 0.002633
0.0044 --- loss: 0.002422
0.0067 --- loss: 0.002820
0.0089 --- loss: 0.004041
0.0111 --- loss: 0.002426
0.0133 --- loss: 0.002383
0.0156 --- loss: 0.002240
0.0178 --- loss: 0.002453
0.0200 --- loss: 0.002750
0.0222 --- loss: 0.001969
0.0244 --- loss: 0.001909
0.0267 --- loss: 0.001907
0.0289 --- loss: 0.001845
0.0311 --- loss: 0.002156
0.0333 --- loss: 0.002472
0.0356 --- loss: 0.001863
0.0378 --- loss: 0.001815
0.0400 --- loss: 0.002063
0.0422 --- loss: 0.002155
0.0444 --- loss: 0.001511
0.0467 --- loss: 0.001683
0.0489 --- loss: 0.002350
0.0511 --- loss: 0.001584
0.0533 --- loss: 0.001540
0.0556 --- loss: 0.001774
0.0578 --- loss: 0.001755
0.0600 --- loss: 0.001474
0.0622 --- loss: 0.001921
0.0644 --- loss: 0.001828
0.0667 --- loss: 0.001819
0.0689 --- loss: 0.001598
0.0711 --- loss: 0.001269
0.0733 --- loss: 0.001637
0.0756 --- loss: 0.001766
0.0778 --- loss: 0.001278
0.0800 --- loss: 0.001634
0.0822 --- loss: 0.001204
0.0844 --- loss: 0.001975
0.0867 --- loss: 0.001994
0.0889 --- loss: 0.001692
0.0911 --- loss: 0.002165
0.0933 --- loss: 0.001584
0.0956 --- loss: 0.001181
0.0978 --- loss: 0.001192
0.1000 --- loss: 0.002082
0.1022 --- loss: 0.001956
0.1044 --- loss: 0.001439
0.1067 --- loss: 0.001061
0.1089 --- loss: 0.001488
0.1111 --- loss: 0.001871
0.1133 --- loss: 0.001303
0.1156 --- loss: 0.001571
0.1178 --- loss: 0.001380
0.1200 --- loss: 0.001611
0.1222 --- loss: 0.001043
0.1244 --- loss: 0.002700
0.1267 --- loss: 0.001413
0.1289 --- loss: 0.001543
0.1311 --- loss: 0.001842
0.1333 --- loss: 0.001404
0.1356 --- loss: 0.001500
0.1378 --- loss: 0.001820
0.1400 --- loss: 0.001596
0.1422 --- loss: 0.001798
0.1444 --- loss: 0.001109
0.1467 --- loss: 0.001387
0.1489 --- loss: 0.001030
0.1511 --- loss: 0.001572
0.1533 --- loss: 0.001960
0.1556 --- loss: 0.001110
0.1578 --- loss: 0.001577
0.1600 --- loss: 0.001843
0.1622 --- loss: 0.001437
0.1644 --- loss: 0.001247
0.1667 --- loss: 0.001126
0.1689 --- loss: 0.001352
0.1711 --- loss: 0.001151
0.1733 --- loss: 0.001233
0.1756 --- loss: 0.001039
0.1778 --- loss: 0.001231
0.1800 --- loss: 0.001592
0.1822 --- loss: 0.001606
0.1844 --- loss: 0.002316
0.1867 --- loss: 0.002292
0.1889 --- loss: 0.001252
0.1911 --- loss: 0.001430
0.1933 --- loss: 0.002370
0.1956 --- loss: 0.001156
0.1978 --- loss: 0.001241
0.2000 --- loss: 0.001221
0.2022 --- loss: 0.001862
0.2044 --- loss: 0.001222
0.2067 --- loss: 0.001464
0.2089 --- loss: 0.001870
0.2111 --- loss: 0.001673
0.2133 --- loss: 0.001756
0.2156 --- loss: 0.001045
0.2178 --- loss: 0.001670
0.2200 --- loss: 0.001592
0.2222 --- loss: 0.001259
0.2244 --- loss: 0.001236
0.2267 --- loss: 0.001007
0.2289 --- loss: 0.001032
0.2311 --- loss: 0.001324
0.2333 --- loss: 0.001021
0.2356 --- loss: 0.000912
0.2378 --- loss: 0.001641
0.2400 --- loss: 0.001441
0.2422 --- loss: 0.001378
0.2444 --- loss: 0.001345
0.2467 --- loss: 0.001766
0.2489 --- loss: 0.001462
0.2511 --- loss: 0.000974
0.2533 --- loss: 0.001049
0.2556 --- loss: 0.001366
0.2578 --- loss: 0.001699
0.2600 --- loss: 0.001164
0.2622 --- loss: 0.001538
0.2644 --- loss: 0.001332
0.2667 --- loss: 0.001437
0.2689 --- loss: 0.001081
0.2711 --- loss: 0.001909
0.2733 --- loss: 0.001515
0.2756 --- loss: 0.000958
0.2778 --- loss: 0.001082
0.2800 --- loss: 0.001981
0.2822 --- loss: 0.001083
0.2844 --- loss: 0.001294
0.2867 --- loss: 0.001787
0.2889 --- loss: 0.001072
0.2911 --- loss: 0.001504
0.2933 --- loss: 0.001234
0.2956 --- loss: 0.001196
0.2978 --- loss: 0.001016
0.3000 --- loss: 0.001172
0.3022 --- loss: 0.001064
0.3044 --- loss: 0.001375
0.3067 --- loss: 0.002056
0.3089 --- loss: 0.001393
0.3111 --- loss: 0.000953
0.3133 --- loss: 0.001219
0.3156 --- loss: 0.001002
0.3178 --- loss: 0.000966
0.3200 --- loss: 0.001199
0.3222 --- loss: 0.001195
0.3244 --- loss: 0.001524
0.3267 --- loss: 0.001559
0.3289 --- loss: 0.001130
0.3311 --- loss: 0.001264
0.3333 --- loss: 0.001109
0.3356 --- loss: 0.001353
0.3378 --- loss: 0.001221
0.3400 --- loss: 0.001040
0.3422 --- loss: 0.000973
0.3444 --- loss: 0.001159
0.3467 --- loss: 0.001368
0.3489 --- loss: 0.000866
0.3511 --- loss: 0.000901
0.3533 --- loss: 0.001334
0.3556 --- loss: 0.001145
0.3578 --- loss: 0.001662
0.3600 --- loss: 0.001623
0.3622 --- loss: 0.001068
0.3644 --- loss: 0.001180
0.3667 --- loss: 0.001662
0.3689 --- loss: 0.001038
0.3711 --- loss: 0.001231
0.3733 --- loss: 0.001159
0.3756 --- loss: 0.001023
0.3778 --- loss: 0.001418
0.3800 --- loss: 0.001108
0.3822 --- loss: 0.001028
0.3844 --- loss: 0.001051
0.3867 --- loss: 0.001040
0.3889 --- loss: 0.000834
0.3911 --- loss: 0.001101
0.3933 --- loss: 0.000849
0.3956 --- loss: 0.000916
0.3978 --- loss: 0.000875
0.4000 --- loss: 0.001178
0.4022 --- loss: 0.000849
0.4044 --- loss: 0.001461
0.4067 --- loss: 0.001307
0.4089 --- loss: 0.000983
0.4111 --- loss: 0.001174
0.4133 --- loss: 0.001107
0.4156 --- loss: 0.001216
0.4178 --- loss: 0.001558
0.4200 --- loss: 0.001109
0.4222 --- loss: 0.001118
0.4244 --- loss: 0.000931
0.4267 --- loss: 0.002112
0.4289 --- loss: 0.001368
0.4311 --- loss: 0.000896
0.4333 --- loss: 0.001240
0.4356 --- loss: 0.000963
0.4378 --- loss: 0.000969
0.4400 --- loss: 0.001113
0.4422 --- loss: 0.001593
0.4444 --- loss: 0.001443
0.4467 --- loss: 0.000938
0.4489 --- loss: 0.001330
0.4511 --- loss: 0.001113
0.4533 --- loss: 0.001570
0.4556 --- loss: 0.000903
0.4578 --- loss: 0.001080
0.4600 --- loss: 0.001063
0.4622 --- loss: 0.000995
0.4644 --- loss: 0.001706
0.4667 --- loss: 0.001202
0.4689 --- loss: 0.001088
0.4711 --- loss: 0.001651
0.4733 --- loss: 0.001271
0.4756 --- loss: 0.001272
0.4778 --- loss: 0.001384
0.4800 --- loss: 0.000899
0.4822 --- loss: 0.001078
0.4844 --- loss: 0.000955
0.4867 --- loss: 0.000868
0.4889 --- loss: 0.001045
0.4911 --- loss: 0.000972
0.4933 --- loss: 0.001058
0.4956 --- loss: 0.001603
0.4978 --- loss: 0.001412
0.5000 --- loss: 0.001692
0.5022 --- loss: 0.001204
0.5044 --- loss: 0.001059
0.5067 --- loss: 0.001146
0.5089 --- loss: 0.001194
0.5111 --- loss: 0.001031
0.5133 --- loss: 0.001328
0.5156 --- loss: 0.001356
0.5178 --- loss: 0.001651
0.5200 --- loss: 0.001382
0.5222 --- loss: 0.001605
0.5244 --- loss: 0.001130
0.5267 --- loss: 0.001733
0.5289 --- loss: 0.001362
0.5311 --- loss: 0.001196
0.5333 --- loss: 0.001065
0.5356 --- loss: 0.001236
0.5378 --- loss: 0.001674
0.5400 --- loss: 0.001302
0.5422 --- loss: 0.001245
0.5444 --- loss: 0.001443
0.5467 --- loss: 0.001770
0.5489 --- loss: 0.000893
0.5511 --- loss: 0.001077
0.5533 --- loss: 0.000815
0.5556 --- loss: 0.001438
0.5578 --- loss: 0.000899
0.5600 --- loss: 0.001176
0.5622 --- loss: 0.000825
0.5644 --- loss: 0.001102
0.5667 --- loss: 0.001301
0.5689 --- loss: 0.001396
0.5711 --- loss: 0.001428
0.5733 --- loss: 0.001067
0.5756 --- loss: 0.001077
0.5778 --- loss: 0.001357
0.5800 --- loss: 0.001482
0.5822 --- loss: 0.001212
0.5844 --- loss: 0.001282
0.5867 --- loss: 0.001586
0.5889 --- loss: 0.001204
0.5911 --- loss: 0.001188
0.5933 --- loss: 0.001025
0.5956 --- loss: 0.000830
0.5978 --- loss: 0.001211
0.6000 --- loss: 0.000980
0.6022 --- loss: 0.001711
0.6044 --- loss: 0.001422
0.6067 --- loss: 0.001122
0.6089 --- loss: 0.000997
0.6111 --- loss: 0.000975
0.6133 --- loss: 0.002070
0.6156 --- loss: 0.001189
0.6178 --- loss: 0.000979
0.6200 --- loss: 0.001128
0.6222 --- loss: 0.000903
0.6244 --- loss: 0.001446
0.6267 --- loss: 0.001801
0.6289 --- loss: 0.001013
0.6311 --- loss: 0.001079
0.6333 --- loss: 0.000929
0.6356 --- loss: 0.000880
0.6378 --- loss: 0.002151
0.6400 --- loss: 0.001343
0.6422 --- loss: 0.000878
0.6444 --- loss: 0.001118
0.6467 --- loss: 0.001658
0.6489 --- loss: 0.001288
0.6511 --- loss: 0.001550
0.6533 --- loss: 0.001090
0.6556 --- loss: 0.001394
0.6578 --- loss: 0.001445
0.6600 --- loss: 0.001328
0.6622 --- loss: 0.001593
0.6644 --- loss: 0.001325
0.6667 --- loss: 0.000853
0.6689 --- loss: 0.001330
0.6711 --- loss: 0.001343
0.6733 --- loss: 0.000805
0.6756 --- loss: 0.000979
0.6778 --- loss: 0.001088
0.6800 --- loss: 0.001510
0.6822 --- loss: 0.001783
0.6844 --- loss: 0.001308
0.6867 --- loss: 0.001239
0.6889 --- loss: 0.001260
0.6911 --- loss: 0.001061
0.6933 --- loss: 0.001049
0.6956 --- loss: 0.000911
0.6978 --- loss: 0.001418
0.7000 --- loss: 0.001015
0.7022 --- loss: 0.000987
0.7044 --- loss: 0.001529
0.7067 --- loss: 0.001254
0.7089 --- loss: 0.001268
0.7111 --- loss: 0.001482
0.7133 --- loss: 0.001469
0.7156 --- loss: 0.001600
0.7178 --- loss: 0.001220
0.7200 --- loss: 0.001092
0.7222 --- loss: 0.001110
0.7244 --- loss: 0.001149
0.7267 --- loss: 0.001372
0.7289 --- loss: 0.001264
0.7311 --- loss: 0.001042
0.7333 --- loss: 0.000986
0.7356 --- loss: 0.001214
0.7378 --- loss: 0.000986
0.7400 --- loss: 0.001299
0.7422 --- loss: 0.001311
0.7444 --- loss: 0.001112
0.7467 --- loss: 0.001130
0.7489 --- loss: 0.001007
0.7511 --- loss: 0.001600
0.7533 --- loss: 0.001115
0.7556 --- loss: 0.001321
0.7578 --- loss: 0.001435
0.7600 --- loss: 0.001255
0.7622 --- loss: 0.000852
0.7644 --- loss: 0.001112
0.7667 --- loss: 0.001117
0.7689 --- loss: 0.000954
0.7711 --- loss: 0.001134
0.7733 --- loss: 0.001492
0.7756 --- loss: 0.001349
0.7778 --- loss: 0.000963
0.7800 --- loss: 0.001478
0.7822 --- loss: 0.001680
0.7844 --- loss: 0.001012
0.7867 --- loss: 0.001065
0.7889 --- loss: 0.001348
0.7911 --- loss: 0.001483
0.7933 --- loss: 0.000934
0.7956 --- loss: 0.001157
0.7978 --- loss: 0.001213
0.8000 --- loss: 0.001099
0.8022 --- loss: 0.000890
0.8044 --- loss: 0.000919
0.8067 --- loss: 0.001519
0.8089 --- loss: 0.001344
0.8111 --- loss: 0.001342
0.8133 --- loss: 0.001802
0.8156 --- loss: 0.001363
0.8178 --- loss: 0.001011
0.8200 --- loss: 0.001204
0.8222 --- loss: 0.001472
0.8244 --- loss: 0.001367
0.8267 --- loss: 0.000806
0.8289 --- loss: 0.000874
0.8311 --- loss: 0.002120
0.8333 --- loss: 0.001009
0.8356 --- loss: 0.001562
0.8378 --- loss: 0.000936
0.8400 --- loss: 0.000814
0.8422 --- loss: 0.001618
0.8444 --- loss: 0.001343
0.8467 --- loss: 0.001262
0.8489 --- loss: 0.001256
0.8511 --- loss: 0.001348
0.8533 --- loss: 0.001962
0.8556 --- loss: 0.001001
0.8578 --- loss: 0.001748
0.8600 --- loss: 0.001247
0.8622 --- loss: 0.001324
0.8644 --- loss: 0.001192
0.8667 --- loss: 0.001294
0.8689 --- loss: 0.001187
0.8711 --- loss: 0.001662
0.8733 --- loss: 0.001582
0.8756 --- loss: 0.001012
0.8778 --- loss: 0.000997
0.8800 --- loss: 0.001056
0.8822 --- loss: 0.000886
0.8844 --- loss: 0.000999
0.8867 --- loss: 0.000879
0.8889 --- loss: 0.001332
0.8911 --- loss: 0.001247
0.8933 --- loss: 0.000965
0.8956 --- loss: 0.001219
0.8978 --- loss: 0.001183
0.9000 --- loss: 0.001097
0.9022 --- loss: 0.001348
0.9044 --- loss: 0.002425
0.9067 --- loss: 0.000825
0.9089 --- loss: 0.001274
0.9111 --- loss: 0.001173
0.9133 --- loss: 0.000822
0.9156 --- loss: 0.001414
0.9178 --- loss: 0.000840
0.9200 --- loss: 0.001693
0.9222 --- loss: 0.001153
0.9244 --- loss: 0.002076
0.9267 --- loss: 0.001016
0.9289 --- loss: 0.001503
0.9311 --- loss: 0.001090
0.9333 --- loss: 0.001189
0.9356 --- loss: 0.000976
0.9378 --- loss: 0.001043
0.9400 --- loss: 0.001589
0.9422 --- loss: 0.001240
0.9444 --- loss: 0.001022
0.9467 --- loss: 0.001346
0.9489 --- loss: 0.001301
0.9511 --- loss: 0.000906
0.9533 --- loss: 0.000913
0.9556 --- loss: 0.000961
0.9578 --- loss: 0.001394
0.9600 --- loss: 0.001018
0.9622 --- loss: 0.001388
0.9644 --- loss: 0.001023
0.9667 --- loss: 0.001472
0.9689 --- loss: 0.001101
0.9711 --- loss: 0.000990
0.9733 --- loss: 0.000993
0.9756 --- loss: 0.000938
0.9778 --- loss: 0.001100
0.9800 --- loss: 0.001063
0.9822 --- loss: 0.001124
0.9844 --- loss: 0.000924
0.9867 --- loss: 0.000883
0.9889 --- loss: 0.001119
0.9911 --- loss: 0.001157
0.9933 --- loss: 0.000928
Epoch finished ! Loss: 0.0013593644612381272
Checkpoint e1 saved !

real	5m27.066s
user	33m33.154s
sys	1m49.445s
