Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/progress.png filter=lfs diff=lfs merge=lfs -text
+Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/progress.png filter=lfs diff=lfs merge=lfs -text
+Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/progress.png filter=lfs diff=lfs merge=lfs -text
+Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/progress.png filter=lfs diff=lfs merge=lfs -text
+Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/progress.png filter=lfs diff=lfs merge=lfs -text
+Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/progress.png filter=lfs diff=lfs merge=lfs -text
+Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_1/progress.png filter=lfs diff=lfs merge=lfs -text
+Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_2/progress.png filter=lfs diff=lfs merge=lfs -text

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/debug.json

@@ -0,0 +1,106 @@
+{
+    "all_tr_losses": "[0.33703446, 0.245874, 0.20384128, 0.19027823, 0.17232056, 0.17565493, 0.15834498, 0.14658068, 0.15074384, 0.15878943, 0.16765323, 0.14644063, 0.14780703, 0.13994478, 0.14089847, 0.13259973, 0.1294894, 0.13168564, 0.13727094, 0.12322069, 0.1391203, 0.139793, 0.13415627, 0.13622817, 0.13856299, 0.12430216, 0.13102552, 0.12888575, 0.12994584, 0.12861799, 0.1136061, 0.12196406, 0.12626949, 0.13798071, 0.1246965, 0.12701035, 0.12746197, 0.1239965, 0.12073733, 0.11822609, 0.11616988, 0.118598886, 0.11856192, 0.119665414, 0.1165078, 0.12099173, 0.11922734, 0.11969286, 0.11703803, 0.115240365, 0.11297671, 0.114790976, 0.10922662, 0.11488171, 0.11453125, 0.10819345, 0.113893956, 0.106148824, 0.11596619, 0.10987344, 0.10946266, 0.11625536, 0.110896274, 0.11666065, 0.11128778, 0.10665962, 0.10384848, 0.10787832, 0.10700743, 0.105068035, 0.10964044, 0.10834642, 0.104676664, 0.10858993, 0.10325108, 0.106033444, 0.10648876, 0.10127316, 0.10753345, 0.103597306, 0.11094735, 0.104656205, 0.10037432, 0.10148334, 0.10345265, 0.10251114, 0.099607326, 0.10200581, 0.098598756, 0.10067512, 0.100201815, 0.09806329, 0.10444377]",
+    "all_val_eval_metrics": "[0.24931305466128356, 0.4266110160090153, 0.5166685259504604, 0.4423168852681748, 0.6139433036241648, 0.6516204582278422, 0.6189222721406125, 0.6413063344985869, 0.665092366906004, 0.6947147104919859, 0.6683744516164448, 0.6736381439436037, 0.700017381967438, 0.6730759539684324, 0.7154415276136535, 0.6694335787092658, 0.7192062580337659, 0.7044392301422628, 0.7385992185846558, 0.7161301051467699, 0.7369110132477408, 0.7564931665840715, 0.7050542082952844, 0.7383295048202763, 0.7363267486563524, 0.7474278933635006, 0.7232240493114174, 0.7471583706697802, 0.7589230466820907, 0.7424222985280224, 0.7350535327813896, 0.7524819191153387, 0.694568537220704, 0.7469446388453539, 0.7381758215499443, 0.737004824372373, 0.7473608181248375, 0.7227166179962762, 0.7555413616261274, 0.7406594892576732, 0.7364759507678018, 0.7484061102330988, 0.7489865782929418, 0.7502880666822354, 0.7605923448265312, 0.7649212228713784, 0.7506574964829025, 0.7653932620891901, 0.7627684916894186, 0.7362864575110082, 0.7246187154809203, 0.7573861141126148, 0.7646790277164011, 0.7577838008902644, 0.7546682915865979, 0.7727560666071309, 0.7416302187811665, 0.7694858066007401, 0.7446922180130513, 0.7420685030243155, 0.7684926564568911, 0.7650307492496151, 0.765980062045606, 0.7519805120045, 0.7735396243562005, 0.7754028667100326, 0.7694532069282431, 0.7717403203125063, 0.7643728398963558, 0.7773234921768635, 0.7689947898983847, 0.7713590280372293, 0.7743324021786255, 0.7604616566054231, 0.7711240265350421, 0.7652490678219319, 0.773581396919216, 0.769286730258646, 0.7612245709294823, 0.770080856652207, 0.7657448581798589, 0.7783318644337596, 0.7799934980280342, 0.7710235992523882, 0.7585932516756204, 0.7591866048825404, 0.7573924081938319, 0.775203482934095, 0.7557587434778792, 0.7669206932986474, 0.7655476706708516, 0.7701122568719798, 0.7673466319562363]",
+    "all_val_losses": "[0.25257468, 0.19878879, 0.17702252, 0.16008532, 0.16101126, 0.14179625, 0.13051416, 0.14159404, 0.1414402, 0.15901689, 0.12986815, 0.12475631, 0.12460776, 0.14963786, 0.12489662, 0.11893177, 0.12789702, 0.12574145, 0.14572573, 0.12154336, 0.13174886, 0.11971721, 0.123625204, 0.12817937, 0.13126977, 0.12630185, 0.12804723, 0.13088135, 0.120637454, 0.12525234, 0.12320873, 0.12474595, 0.12430963, 0.11961495, 0.119082205, 0.1295736, 0.114206515, 0.1255392, 0.12905027, 0.11628304, 0.11136678, 0.10540939, 0.12599255, 0.116712674, 0.118062906, 0.13245945, 0.120466255, 0.111738354, 0.13182618, 0.11221204, 0.12556745, 0.12311455, 0.13487303, 0.11514746, 0.11674822, 0.115926884, 0.09430425, 0.119221926, 0.11627844, 0.111686766, 0.101318166, 0.10261526, 0.127733, 0.118940614, 0.11854798, 0.10676561, 0.107232295, 0.09977675, 0.10480553, 0.100758284, 0.1097453, 0.11992353, 0.10713313, 0.10082048, 0.10904568, 0.118674114, 0.11127121, 0.098175794, 0.11891383, 0.111774474, 0.10249209, 0.11591384, 0.10567763, 0.10943615, 0.117287934, 0.13489087, 0.107407406, 0.09537521, 0.09355242, 0.10312289, 0.10185069, 0.110794134, 0.1276715]",
+    "all_val_losses_tr_mode": "[]",
+    "also_val_in_tr_mode": "False",
+    "amp_grad_scaler": "<torch.cuda.amp.grad_scaler.GradScaler object at 0x7f29e4484588>",
+    "args": "Namespace(batch_size=2, config='./configs/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32.yaml', continue_training=False, crop_size=[64, 192, 192], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='0', fp32=False, hdfs_base='GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=125, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task008_HepaticVessel', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, vallatest=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)",
+    "base_num_features": "32",
+    "basic_generator_patch_size": "[178 308 233]",
+    "batch_dice": "True",
+    "batch_size": "2",
+    "best_MA_tr_loss_for_patience": "None",
+    "best_epoch_based_on_MA_tr_loss": "None",
+    "best_val_eval_criterion_MA": "None",
+    "classes": "[1, 2]",
+    "conv_per_stage": "2",
+    "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-0.5235987755982988, 0.5235987755982988), 'rotation_y': (-0.5235987755982988, 0.5235987755982988), 'rotation_z': (-0.5235987755982988, 0.5235987755982988), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([ 64, 192, 192])}",
+    "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel",
+    "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]",
+    "deterministic": "False",
+    "disable_ds": "True",
+    "distribute_batch_size": "False",
+    "dl_tr": "<dataset_loading.DataLoader3D object at 0x7f2a32538ef0>",
+    "dl_val": "<dataset_loading.DataLoader3D object at 0x7f2a402ebdd8>",
+    "do_dummy_2D_aug": "False",
+    "ds_loss_weights": "[1. 0. 0. 0. 0.]",
+    "epoch": "93",
+    "experiment_name": "nnUNetTrainerV2_DDP",
+    "find_zero_weight_decay": "False",
+    "fold": "0",
+    "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetData_plans_v2.1_stage1",
+    "fp16": "True",
+    "global_batch_size": "16",
+    "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/gt_segmentations",
+    "inference_pad_border_mode": "constant",
+    "inference_pad_kwargs": "{'constant_values': 0}",
+    "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl', 0, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', '/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel', True, 1, True, False, False, True)",
+    "initial_lr": "0.0003",
+    "input_size": "[64, 192, 192]",
+    "is_spatial_aug_only": "False",
+    "is_ssl": "False",
+    "layer_decay": "1.0",
+    "local_rank": "0",
+    "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/training_log_2022_5_19_23_48_51.txt",
+    "loss": "None",
+    "lr_scheduler": "<optimizers.lr_scheduler.LinearWarmupCosineAnnealingLR object at 0x7f2a31ac34e0>",
+    "lr_scheduler_eps": "0.001",
+    "lr_scheduler_name": "warmup_cosine",
+    "lr_scheduler_patience": "30",
+    "lr_threshold": "1e-06",
+    "max_num_epochs": "125",
+    "min_lr": "0",
+    "min_region_size_per_class": "None",
+    "min_size_per_class": "None",
+    "model": "Generic_TransUNet_max_ppbp",
+    "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}",
+    "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]",
+    "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]]",
+    "net_pool_per_axis": "[4, 5, 5]",
+    "normalization_schemes": "OrderedDict([(0, 'CT')])",
+    "num_batches_per_epoch": "250",
+    "num_classes": "3",
+    "num_input_channels": "1",
+    "num_val_batches_per_epoch": "50",
+    "online_eval_fn": "[]",
+    "online_eval_foreground_dc": "[]",
+    "online_eval_fp": "[]",
+    "online_eval_tp": "[]",
+    "only_keep_largest_connected_component": "None",
+    "optim_name": "adamw",
+    "optimizer": "AdamW (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    eps: 1e-08\n    initial_lr: 0.0003\n    lr: 5.375636079717583e-05\n    weight_decay: 3e-05\n)",
+    "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0",
+    "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32",
+    "oversample_foreground_percent": "0.0",
+    "pad_all_sides": "None",
+    "patch_size": "[ 64 192 192]",
+    "patience": "50",
+    "pin_memory": "True",
+    "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl",
+    "reclip": "None",
+    "regions_class_order": "None",
+    "resume": "auto",
+    "save_best_checkpoint": "True",
+    "save_every": "1",
+    "save_final_checkpoint": "True",
+    "save_intermediate_checkpoints": "True",
+    "save_latest_only": "True",
+    "stage": "1",
+    "threeD": "True",
+    "tr_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7f2a32442518>",
+    "train_loss_MA": "None",
+    "train_loss_MA_alpha": "0.93",
+    "train_loss_MA_eps": "0.0005",
+    "transpose_backward": "[0, 1, 2]",
+    "transpose_forward": "[0, 1, 2]",
+    "unpack_data": "True",
+    "use_mask_for_norm": "OrderedDict([(0, False)])",
+    "use_progress_bar": "False",
+    "val_eval_criterion_MA": "None",
+    "val_eval_criterion_alpha": "0.9",
+    "val_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7f2a324423c8>",
+    "warmup_epochs": "10",
+    "was_initialized": "True",
+    "weight_decay": "3e-05"
+}

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_best.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acc3c382ff646fa41dd75eeea3490c0b929199e3b52533a7c4d5144c7e6ee02e
+size 496527929

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_best.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7e84f1583a7b2b7d0ff43ee2d2c257d6c2589715299940df967ac18db1196c6
+size 188932

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_final_checkpoint.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ca558721b5616594b3c4006cfc12be201310d5270b80fbbb903fa62dde0d575
+size 496529594

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_final_checkpoint.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7e84f1583a7b2b7d0ff43ee2d2c257d6c2589715299940df967ac18db1196c6
+size 188932

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_latest.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:528cc64b7f13d252ed7709c0a8761b69327ba81eaf89d5dc83248fd11f63a4ed
+size 496529594

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_latest.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7e84f1583a7b2b7d0ff43ee2d2c257d6c2589715299940df967ac18db1196c6
+size 188932

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/training_log_2022_5_19_23_48_51.txt

@@ -0,0 +1,459 @@
+Starting... 
+2022-05-19 23:48:51.257375: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/splits_final.pkl 
+2022-05-19 23:48:51.257992: The split file contains 5 splits. 
+2022-05-19 23:48:51.258045: Desired fold for training: 0 
+2022-05-19 23:48:51.258093: This split has 242 training and 61 validation cases. 
+2022-05-19 23:50:29.435884: TRAINING KEYS:
+ odict_keys(['hepaticvessel_001', 'hepaticvessel_002', 'hepaticvessel_004', 'hepaticvessel_005', 'hepaticvessel_007', 'hepaticvessel_008', 'hepaticvessel_010', 'hepaticvessel_011', 'hepaticvessel_013', 'hepaticvessel_016', 'hepaticvessel_020', 'hepaticvessel_022', 'hepaticvessel_023', 'hepaticvessel_025', 'hepaticvessel_027', 'hepaticvessel_029', 'hepaticvessel_031', 'hepaticvessel_032', 'hepaticvessel_033', 'hepaticvessel_039', 'hepaticvessel_040', 'hepaticvessel_042', 'hepaticvessel_044', 'hepaticvessel_046', 'hepaticvessel_049', 'hepaticvessel_051', 'hepaticvessel_052', 'hepaticvessel_057', 'hepaticvessel_059', 'hepaticvessel_061', 'hepaticvessel_062', 'hepaticvessel_065', 'hepaticvessel_067', 'hepaticvessel_068', 'hepaticvessel_070', 'hepaticvessel_071', 'hepaticvessel_072', 'hepaticvessel_075', 'hepaticvessel_078', 'hepaticvessel_079', 'hepaticvessel_081', 'hepaticvessel_083', 'hepaticvessel_086', 'hepaticvessel_088', 'hepaticvessel_089', 'hepaticvessel_090', 'hepaticvessel_091', 'hepaticvessel_092', 'hepaticvessel_093', 'hepaticvessel_094', 'hepaticvessel_096', 'hepaticvessel_097', 'hepaticvessel_102', 'hepaticvessel_103', 'hepaticvessel_104', 'hepaticvessel_108', 'hepaticvessel_111', 'hepaticvessel_113', 'hepaticvessel_115', 'hepaticvessel_116', 'hepaticvessel_117', 'hepaticvessel_119', 'hepaticvessel_121', 'hepaticvessel_123', 'hepaticvessel_124', 'hepaticvessel_125', 'hepaticvessel_128', 'hepaticvessel_131', 'hepaticvessel_132', 'hepaticvessel_133', 'hepaticvessel_136', 'hepaticvessel_137', 'hepaticvessel_138', 'hepaticvessel_139', 'hepaticvessel_140', 'hepaticvessel_141', 'hepaticvessel_142', 'hepaticvessel_144', 'hepaticvessel_145', 'hepaticvessel_148', 'hepaticvessel_151', 'hepaticvessel_154', 'hepaticvessel_158', 'hepaticvessel_159', 'hepaticvessel_160', 'hepaticvessel_163', 'hepaticvessel_165', 'hepaticvessel_166', 'hepaticvessel_171', 'hepaticvessel_172', 'hepaticvessel_173', 'hepaticvessel_174', 'hepaticvessel_177', 'hepaticvessel_178', 'hepaticvessel_179', 'hepaticvessel_181', 'hepaticvessel_183', 'hepaticvessel_184', 'hepaticvessel_185', 'hepaticvessel_186', 'hepaticvessel_189', 'hepaticvessel_192', 'hepaticvessel_193', 'hepaticvessel_195', 'hepaticvessel_196', 'hepaticvessel_197', 'hepaticvessel_198', 'hepaticvessel_199', 'hepaticvessel_200', 'hepaticvessel_201', 'hepaticvessel_203', 'hepaticvessel_207', 'hepaticvessel_208', 'hepaticvessel_209', 'hepaticvessel_210', 'hepaticvessel_211', 'hepaticvessel_214', 'hepaticvessel_215', 'hepaticvessel_219', 'hepaticvessel_221', 'hepaticvessel_222', 'hepaticvessel_223', 'hepaticvessel_225', 'hepaticvessel_227', 'hepaticvessel_229', 'hepaticvessel_230', 'hepaticvessel_231', 'hepaticvessel_233', 'hepaticvessel_235', 'hepaticvessel_236', 'hepaticvessel_237', 'hepaticvessel_238', 'hepaticvessel_240', 'hepaticvessel_242', 'hepaticvessel_245', 'hepaticvessel_246', 'hepaticvessel_255', 'hepaticvessel_256', 'hepaticvessel_258', 'hepaticvessel_259', 'hepaticvessel_262', 'hepaticvessel_263', 'hepaticvessel_265', 'hepaticvessel_268', 'hepaticvessel_271', 'hepaticvessel_278', 'hepaticvessel_279', 'hepaticvessel_280', 'hepaticvessel_281', 'hepaticvessel_284', 'hepaticvessel_285', 'hepaticvessel_286', 'hepaticvessel_287', 'hepaticvessel_289', 'hepaticvessel_290', 'hepaticvessel_291', 'hepaticvessel_293', 'hepaticvessel_294', 'hepaticvessel_296', 'hepaticvessel_297', 'hepaticvessel_299', 'hepaticvessel_300', 'hepaticvessel_303', 'hepaticvessel_305', 'hepaticvessel_307', 'hepaticvessel_308', 'hepaticvessel_314', 'hepaticvessel_318', 'hepaticvessel_319', 'hepaticvessel_320', 'hepaticvessel_321', 'hepaticvessel_322', 'hepaticvessel_323', 'hepaticvessel_324', 'hepaticvessel_325', 'hepaticvessel_327', 'hepaticvessel_330', 'hepaticvessel_332', 'hepaticvessel_335', 'hepaticvessel_336', 'hepaticvessel_338', 'hepaticvessel_340', 'hepaticvessel_341', 'hepaticvessel_344', 'hepaticvessel_345', 'hepaticvessel_348', 'hepaticvessel_349', 'hepaticvessel_350', 'hepaticvessel_353', 'hepaticvessel_356', 'hepaticvessel_362', 'hepaticvessel_367', 'hepaticvessel_369', 'hepaticvessel_371', 'hepaticvessel_372', 'hepaticvessel_373', 'hepaticvessel_374', 'hepaticvessel_378', 'hepaticvessel_379', 'hepaticvessel_382', 'hepaticvessel_383', 'hepaticvessel_384', 'hepaticvessel_385', 'hepaticvessel_386', 'hepaticvessel_388', 'hepaticvessel_389', 'hepaticvessel_391', 'hepaticvessel_393', 'hepaticvessel_396', 'hepaticvessel_397', 'hepaticvessel_398', 'hepaticvessel_400', 'hepaticvessel_401', 'hepaticvessel_402', 'hepaticvessel_404', 'hepaticvessel_406', 'hepaticvessel_407', 'hepaticvessel_408', 'hepaticvessel_409', 'hepaticvessel_411', 'hepaticvessel_412', 'hepaticvessel_416', 'hepaticvessel_419', 'hepaticvessel_420', 'hepaticvessel_423', 'hepaticvessel_425', 'hepaticvessel_427', 'hepaticvessel_429', 'hepaticvessel_431', 'hepaticvessel_432', 'hepaticvessel_433', 'hepaticvessel_434', 'hepaticvessel_438', 'hepaticvessel_440', 'hepaticvessel_442', 'hepaticvessel_443', 'hepaticvessel_446', 'hepaticvessel_447', 'hepaticvessel_453', 'hepaticvessel_454', 'hepaticvessel_455', 'hepaticvessel_456']) 
+2022-05-19 23:50:29.436077: VALIDATION KEYS:
+ odict_keys(['hepaticvessel_018', 'hepaticvessel_019', 'hepaticvessel_026', 'hepaticvessel_028', 'hepaticvessel_030', 'hepaticvessel_050', 'hepaticvessel_053', 'hepaticvessel_058', 'hepaticvessel_066', 'hepaticvessel_077', 'hepaticvessel_080', 'hepaticvessel_082', 'hepaticvessel_084', 'hepaticvessel_085', 'hepaticvessel_087', 'hepaticvessel_098', 'hepaticvessel_100', 'hepaticvessel_110', 'hepaticvessel_112', 'hepaticvessel_127', 'hepaticvessel_129', 'hepaticvessel_146', 'hepaticvessel_147', 'hepaticvessel_150', 'hepaticvessel_157', 'hepaticvessel_161', 'hepaticvessel_164', 'hepaticvessel_167', 'hepaticvessel_175', 'hepaticvessel_194', 'hepaticvessel_206', 'hepaticvessel_213', 'hepaticvessel_217', 'hepaticvessel_218', 'hepaticvessel_224', 'hepaticvessel_234', 'hepaticvessel_244', 'hepaticvessel_248', 'hepaticvessel_266', 'hepaticvessel_269', 'hepaticvessel_270', 'hepaticvessel_272', 'hepaticvessel_274', 'hepaticvessel_275', 'hepaticvessel_282', 'hepaticvessel_309', 'hepaticvessel_329', 'hepaticvessel_333', 'hepaticvessel_358', 'hepaticvessel_359', 'hepaticvessel_361', 'hepaticvessel_363', 'hepaticvessel_368', 'hepaticvessel_375', 'hepaticvessel_422', 'hepaticvessel_424', 'hepaticvessel_437', 'hepaticvessel_441', 'hepaticvessel_444', 'hepaticvessel_445', 'hepaticvessel_458']) 
+2022-05-19 23:50:39.562442: loading checkpoint /opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_0/model_latest.model train= True 
+2022-05-19 23:50:40.782292: lr: 5.1e-05 
+2022-05-19 23:50:54.898852: 
+epoch:  93 
+2022-05-19 23:58:50.384474: train loss : 0.1051 
+2022-05-19 23:59:16.686494: validation loss: 0.1173 
+2022-05-19 23:59:16.687123: Average global foreground Dice: [0.6786, 0.8481] 
+2022-05-19 23:59:16.687212: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 23:59:17.112055: lr: 5.1e-05 
+2022-05-19 23:59:17.112260: saving scheduled checkpoint file... 
+2022-05-19 23:59:17.207761: saving checkpoint... 
+2022-05-19 23:59:18.036854: done, saving took 0.92 seconds 
+2022-05-19 23:59:18.044720: done 
+2022-05-19 23:59:32.595264: This epoch took 517.695660 s
+ 
+2022-05-19 23:59:32.595533: 
+epoch:  94 
+2022-05-20 00:07:15.572337: train loss : 0.1075 
+2022-05-20 00:07:41.835408: validation loss: 0.0935 
+2022-05-20 00:07:41.835972: Average global foreground Dice: [0.6847, 0.8776] 
+2022-05-20 00:07:41.836137: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:07:42.239912: lr: 4.8e-05 
+2022-05-20 00:07:42.240106: saving scheduled checkpoint file... 
+2022-05-20 00:07:42.298090: saving checkpoint... 
+2022-05-20 00:07:43.182063: done, saving took 0.94 seconds 
+2022-05-20 00:07:43.191494: done 
+2022-05-20 00:07:43.231501: saving checkpoint... 
+2022-05-20 00:07:43.832907: done, saving took 0.64 seconds 
+2022-05-20 00:08:05.325558: This epoch took 512.729770 s
+ 
+2022-05-20 00:08:05.325897: 
+epoch:  95 
+2022-05-20 00:15:47.981698: train loss : 0.0986 
+2022-05-20 00:16:14.300343: validation loss: 0.1020 
+2022-05-20 00:16:14.300973: Average global foreground Dice: [0.6792, 0.8661] 
+2022-05-20 00:16:14.301059: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:16:14.702829: lr: 4.5e-05 
+2022-05-20 00:16:14.703036: saving scheduled checkpoint file... 
+2022-05-20 00:16:14.761148: saving checkpoint... 
+2022-05-20 00:16:15.674159: done, saving took 0.97 seconds 
+2022-05-20 00:16:15.684278: done 
+2022-05-20 00:16:15.725320: saving checkpoint... 
+2022-05-20 00:16:16.609183: done, saving took 0.92 seconds 
+2022-05-20 00:17:02.944608: This epoch took 537.618602 s
+ 
+2022-05-20 00:17:02.944928: 
+epoch:  96 
+2022-05-20 00:24:45.448443: train loss : 0.1029 
+2022-05-20 00:25:11.735332: validation loss: 0.1024 
+2022-05-20 00:25:11.735900: Average global foreground Dice: [0.6728, 0.8555] 
+2022-05-20 00:25:11.735984: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:25:12.136425: lr: 4.2e-05 
+2022-05-20 00:25:12.136624: saving scheduled checkpoint file... 
+2022-05-20 00:25:12.193799: saving checkpoint... 
+2022-05-20 00:25:13.058001: done, saving took 0.92 seconds 
+2022-05-20 00:25:13.066621: done 
+2022-05-20 00:25:36.029841: This epoch took 513.084797 s
+ 
+2022-05-20 00:25:36.030130: 
+epoch:  97 
+2022-05-20 00:33:18.608969: train loss : 0.0923 
+2022-05-20 00:33:44.970985: validation loss: 0.1178 
+2022-05-20 00:33:44.971601: Average global foreground Dice: [0.6633, 0.8634] 
+2022-05-20 00:33:44.971686: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:33:45.375843: lr: 3.9e-05 
+2022-05-20 00:33:45.376031: saving scheduled checkpoint file... 
+2022-05-20 00:33:45.433744: saving checkpoint... 
+2022-05-20 00:33:46.338244: done, saving took 0.96 seconds 
+2022-05-20 00:33:46.347102: done 
+2022-05-20 00:34:15.189025: This epoch took 519.158788 s
+ 
+2022-05-20 00:34:15.189291: 
+epoch:  98 
+2022-05-20 00:41:58.007195: train loss : 0.0986 
+2022-05-20 00:42:24.339576: validation loss: 0.1003 
+2022-05-20 00:42:24.340255: Average global foreground Dice: [0.6778, 0.8453] 
+2022-05-20 00:42:24.340338: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:42:24.743111: lr: 3.6e-05 
+2022-05-20 00:42:24.743304: saving scheduled checkpoint file... 
+2022-05-20 00:42:24.806367: saving checkpoint... 
+2022-05-20 00:42:25.714694: done, saving took 0.97 seconds 
+2022-05-20 00:42:25.724246: done 
+2022-05-20 00:42:50.617247: This epoch took 515.427840 s
+ 
+2022-05-20 00:42:50.617600: 
+epoch:  99 
+2022-05-20 00:50:33.546897: train loss : 0.0886 
+2022-05-20 00:50:59.868541: validation loss: 0.0985 
+2022-05-20 00:50:59.869164: Average global foreground Dice: [0.6767, 0.8476] 
+2022-05-20 00:50:59.869246: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:51:00.272167: lr: 3.4e-05 
+2022-05-20 00:51:00.272364: saving scheduled checkpoint file... 
+2022-05-20 00:51:00.334502: saving checkpoint... 
+2022-05-20 00:51:01.176983: done, saving took 0.90 seconds 
+2022-05-20 00:51:01.185279: done 
+2022-05-20 00:51:28.496241: This epoch took 517.878529 s
+ 
+2022-05-20 00:51:28.496536: 
+epoch:  100 
+2022-05-20 00:59:11.134774: train loss : 0.0890 
+2022-05-20 00:59:37.465693: validation loss: 0.1076 
+2022-05-20 00:59:37.466276: Average global foreground Dice: [0.6876, 0.8667] 
+2022-05-20 00:59:37.466359: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:59:37.874352: lr: 3.1e-05 
+2022-05-20 00:59:37.874558: saving scheduled checkpoint file... 
+2022-05-20 00:59:37.932466: saving checkpoint... 
+2022-05-20 00:59:38.848706: done, saving took 0.97 seconds 
+2022-05-20 00:59:38.857605: done 
+2022-05-20 00:59:38.894561: saving checkpoint... 
+2022-05-20 00:59:39.768356: done, saving took 0.91 seconds 
+2022-05-20 01:00:17.246230: This epoch took 528.749581 s
+ 
+2022-05-20 01:00:17.246567: 
+epoch:  101 
+2022-05-20 01:07:59.731180: train loss : 0.0926 
+2022-05-20 01:08:26.038965: validation loss: 0.1043 
+2022-05-20 01:08:26.039565: Average global foreground Dice: [0.6687, 0.8613] 
+2022-05-20 01:08:26.039651: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:08:26.511847: lr: 2.9e-05 
+2022-05-20 01:08:26.512047: saving scheduled checkpoint file... 
+2022-05-20 01:08:26.569945: saving checkpoint... 
+2022-05-20 01:08:27.485320: done, saving took 0.97 seconds 
+2022-05-20 01:08:27.495139: done 
+2022-05-20 01:08:49.330272: This epoch took 512.083585 s
+ 
+2022-05-20 01:08:49.330897: 
+epoch:  102 
+2022-05-20 01:16:31.882948: train loss : 0.0988 
+2022-05-20 01:16:58.200999: validation loss: 0.1163 
+2022-05-20 01:16:58.201947: Average global foreground Dice: [0.6942, 0.8482] 
+2022-05-20 01:16:58.202044: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:16:58.609138: lr: 2.6e-05 
+2022-05-20 01:16:58.609334: saving scheduled checkpoint file... 
+2022-05-20 01:16:58.666744: saving checkpoint... 
+2022-05-20 01:16:59.502350: done, saving took 0.89 seconds 
+2022-05-20 01:16:59.511892: done 
+2022-05-20 01:16:59.557263: saving checkpoint... 
+2022-05-20 01:17:00.325649: done, saving took 0.81 seconds 
+2022-05-20 01:19:55.707306: This epoch took 666.376227 s
+ 
+2022-05-20 01:19:55.707617: 
+epoch:  103 
+2022-05-20 01:27:38.293492: train loss : 0.0893 
+2022-05-20 01:28:04.627843: validation loss: 0.1044 
+2022-05-20 01:28:04.628515: Average global foreground Dice: [0.6755, 0.8558] 
+2022-05-20 01:28:04.628602: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:28:05.030528: lr: 2.4e-05 
+2022-05-20 01:28:05.030717: saving scheduled checkpoint file... 
+2022-05-20 01:28:05.088472: saving checkpoint... 
+2022-05-20 01:28:05.999219: done, saving took 0.97 seconds 
+2022-05-20 01:28:06.007967: done 
+2022-05-20 01:29:01.742134: This epoch took 546.034411 s
+ 
+2022-05-20 01:29:01.742395: 
+epoch:  104 
+2022-05-20 01:36:44.285238: train loss : 0.0977 
+2022-05-20 01:37:10.613660: validation loss: 0.1055 
+2022-05-20 01:37:10.614282: Average global foreground Dice: [0.6763, 0.8733] 
+2022-05-20 01:37:10.614366: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:37:11.165598: lr: 2.2e-05 
+2022-05-20 01:37:11.165834: saving scheduled checkpoint file... 
+2022-05-20 01:37:11.228286: saving checkpoint... 
+2022-05-20 01:37:12.072917: done, saving took 0.91 seconds 
+2022-05-20 01:37:12.082669: done 
+2022-05-20 01:37:12.119464: saving checkpoint... 
+2022-05-20 01:37:12.945687: done, saving took 0.86 seconds 
+2022-05-20 01:37:37.482560: This epoch took 515.740038 s
+ 
+2022-05-20 01:37:37.482852: 
+epoch:  105 
+2022-05-20 01:45:20.472046: train loss : 0.1015 
+2022-05-20 01:45:46.792297: validation loss: 0.1087 
+2022-05-20 01:45:46.793066: Average global foreground Dice: [0.6842, 0.8695] 
+2022-05-20 01:45:46.793154: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:45:47.268014: lr: 2e-05 
+2022-05-20 01:45:47.268207: saving scheduled checkpoint file... 
+2022-05-20 01:45:47.325853: saving checkpoint... 
+2022-05-20 01:45:48.240048: done, saving took 0.97 seconds 
+2022-05-20 01:45:48.249109: done 
+2022-05-20 01:45:48.285815: saving checkpoint... 
+2022-05-20 01:45:49.126188: done, saving took 0.88 seconds 
+2022-05-20 01:48:16.299839: This epoch took 638.816876 s
+ 
+2022-05-20 01:48:16.300112: 
+epoch:  106 
+2022-05-20 01:55:59.132738: train loss : 0.0985 
+2022-05-20 01:56:25.476089: validation loss: 0.0985 
+2022-05-20 01:56:25.476704: Average global foreground Dice: [0.6772, 0.8671] 
+2022-05-20 01:56:25.476788: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:56:25.884570: lr: 1.8e-05 
+2022-05-20 01:56:25.884767: saving scheduled checkpoint file... 
+2022-05-20 01:56:25.942616: saving checkpoint... 
+2022-05-20 01:56:26.860924: done, saving took 0.98 seconds 
+2022-05-20 01:56:26.870733: done 
+2022-05-20 01:56:26.920466: saving checkpoint... 
+2022-05-20 01:56:27.852342: done, saving took 0.98 seconds 
+2022-05-20 01:56:47.607734: This epoch took 511.307517 s
+ 
+2022-05-20 01:56:47.608024: 
+epoch:  107 
+2022-05-20 02:04:31.711817: train loss : 0.1004 
+2022-05-20 02:04:58.039280: validation loss: 0.1094 
+2022-05-20 02:04:58.039928: Average global foreground Dice: [0.6832, 0.8575] 
+2022-05-20 02:04:58.040011: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:04:58.442301: lr: 1.6e-05 
+2022-05-20 02:04:58.442496: saving scheduled checkpoint file... 
+2022-05-20 02:04:58.500450: saving checkpoint... 
+2022-05-20 02:04:59.355402: done, saving took 0.91 seconds 
+2022-05-20 02:04:59.364923: done 
+2022-05-20 02:04:59.404097: saving checkpoint... 
+2022-05-20 02:05:00.236540: done, saving took 0.87 seconds 
+2022-05-20 02:05:19.533822: This epoch took 511.925693 s
+ 
+2022-05-20 02:05:19.534085: 
+epoch:  108 
+2022-05-20 02:13:02.235750: train loss : 0.1050 
+2022-05-20 02:13:28.574315: validation loss: 0.1035 
+2022-05-20 02:13:28.575045: Average global foreground Dice: [0.6736, 0.8646] 
+2022-05-20 02:13:28.575130: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:13:28.977365: lr: 1.4e-05 
+2022-05-20 02:13:28.977578: saving scheduled checkpoint file... 
+2022-05-20 02:13:29.035761: saving checkpoint... 
+2022-05-20 02:13:29.966818: done, saving took 0.99 seconds 
+2022-05-20 02:13:29.975782: done 
+2022-05-20 02:13:30.013183: saving checkpoint... 
+2022-05-20 02:13:30.904726: done, saving took 0.93 seconds 
+2022-05-20 02:14:20.017928: This epoch took 540.483616 s
+ 
+2022-05-20 02:14:20.018280: 
+epoch:  109 
+2022-05-20 02:22:02.996914: train loss : 0.0922 
+2022-05-20 02:22:29.338303: validation loss: 0.1069 
+2022-05-20 02:22:29.338900: Average global foreground Dice: [0.6791, 0.8653] 
+2022-05-20 02:22:29.338984: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:22:29.741534: lr: 1.2e-05 
+2022-05-20 02:22:29.741723: saving scheduled checkpoint file... 
+2022-05-20 02:22:29.799325: saving checkpoint... 
+2022-05-20 02:22:30.730340: done, saving took 0.99 seconds 
+2022-05-20 02:22:30.753898: done 
+2022-05-20 02:22:30.791040: saving checkpoint... 
+2022-05-20 02:22:31.693027: done, saving took 0.94 seconds 
+2022-05-20 02:23:19.686598: This epoch took 539.668195 s
+ 
+2022-05-20 02:23:19.686899: 
+epoch:  110 
+2022-05-20 02:31:02.481437: train loss : 0.0976 
+2022-05-20 02:31:28.836215: validation loss: 0.1074 
+2022-05-20 02:31:28.836871: Average global foreground Dice: [0.6729, 0.8662] 
+2022-05-20 02:31:28.836956: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:31:29.247230: lr: 1.1e-05 
+2022-05-20 02:31:29.247439: saving scheduled checkpoint file... 
+2022-05-20 02:31:29.305346: saving checkpoint... 
+2022-05-20 02:31:30.255769: done, saving took 1.01 seconds 
+2022-05-20 02:31:30.264834: done 
+2022-05-20 02:31:30.301665: saving checkpoint... 
+2022-05-20 02:31:31.221288: done, saving took 0.96 seconds 
+2022-05-20 02:31:59.297751: This epoch took 519.610735 s
+ 
+2022-05-20 02:31:59.298295: 
+epoch:  111 
+2022-05-20 02:39:42.628542: train loss : 0.0952 
+2022-05-20 02:40:08.969127: validation loss: 0.1235 
+2022-05-20 02:40:08.969854: Average global foreground Dice: [0.6641, 0.8732] 
+2022-05-20 02:40:08.969939: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:40:09.375850: lr: 9e-06 
+2022-05-20 02:40:09.376051: saving scheduled checkpoint file... 
+2022-05-20 02:40:09.433630: saving checkpoint... 
+2022-05-20 02:40:10.268741: done, saving took 0.89 seconds 
+2022-05-20 02:40:10.281483: done 
+2022-05-20 02:40:44.673769: This epoch took 525.375242 s
+ 
+2022-05-20 02:40:44.674232: 
+epoch:  112 
+2022-05-20 02:48:27.491620: train loss : 0.0917 
+2022-05-20 02:48:53.825438: validation loss: 0.1077 
+2022-05-20 02:48:53.826009: Average global foreground Dice: [0.6779, 0.8763] 
+2022-05-20 02:48:53.826092: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:48:54.231390: lr: 8e-06 
+2022-05-20 02:48:54.231585: saving scheduled checkpoint file... 
+2022-05-20 02:48:54.288993: saving checkpoint... 
+2022-05-20 02:48:55.167226: done, saving took 0.94 seconds 
+2022-05-20 02:48:55.175916: done 
+2022-05-20 02:48:55.211892: saving checkpoint... 
+2022-05-20 02:48:56.000624: done, saving took 0.82 seconds 
+2022-05-20 02:49:16.572285: This epoch took 511.897707 s
+ 
+2022-05-20 02:49:16.572624: 
+epoch:  113 
+2022-05-20 02:56:59.628493: train loss : 0.0952 
+2022-05-20 02:57:25.966178: validation loss: 0.1075 
+2022-05-20 02:57:25.966736: Average global foreground Dice: [0.679, 0.8666] 
+2022-05-20 02:57:25.966820: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:57:26.374386: lr: 7e-06 
+2022-05-20 02:57:26.374579: saving scheduled checkpoint file... 
+2022-05-20 02:57:26.432114: saving checkpoint... 
+2022-05-20 02:57:27.399751: done, saving took 1.03 seconds 
+2022-05-20 02:57:27.409125: done 
+2022-05-20 02:57:27.446420: saving checkpoint... 
+2022-05-20 02:57:28.369764: done, saving took 0.96 seconds 
+2022-05-20 02:57:52.518135: This epoch took 515.945405 s
+ 
+2022-05-20 02:57:52.518406: 
+epoch:  114 
+2022-05-20 03:05:35.819391: train loss : 0.0866 
+2022-05-20 03:06:02.155639: validation loss: 0.1065 
+2022-05-20 03:06:02.156239: Average global foreground Dice: [0.6804, 0.8632] 
+2022-05-20 03:06:02.156324: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:06:02.561257: lr: 6e-06 
+2022-05-20 03:06:02.561461: saving scheduled checkpoint file... 
+2022-05-20 03:06:02.619571: saving checkpoint... 
+2022-05-20 03:06:03.557004: done, saving took 1.00 seconds 
+2022-05-20 03:06:03.566626: done 
+2022-05-20 03:06:03.607573: saving checkpoint... 
+2022-05-20 03:06:04.532976: done, saving took 0.97 seconds 
+2022-05-20 03:06:36.565466: This epoch took 524.046920 s
+ 
+2022-05-20 03:06:36.565741: 
+epoch:  115 
+2022-05-20 03:14:19.249373: train loss : 0.0968 
+2022-05-20 03:14:45.599937: validation loss: 0.1125 
+2022-05-20 03:14:45.600540: Average global foreground Dice: [0.6752, 0.8804] 
+2022-05-20 03:14:45.600625: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:14:46.007247: lr: 5e-06 
+2022-05-20 03:14:46.007459: saving scheduled checkpoint file... 
+2022-05-20 03:14:46.064964: saving checkpoint... 
+2022-05-20 03:14:47.052503: done, saving took 1.04 seconds 
+2022-05-20 03:14:47.062558: done 
+2022-05-20 03:14:47.103672: saving checkpoint... 
+2022-05-20 03:14:48.029160: done, saving took 0.97 seconds 
+2022-05-20 03:15:08.900486: This epoch took 512.334484 s
+ 
+2022-05-20 03:15:08.900937: 
+epoch:  116 
+2022-05-20 03:22:51.727525: train loss : 0.0989 
+2022-05-20 03:23:18.064680: validation loss: 0.1056 
+2022-05-20 03:23:18.065268: Average global foreground Dice: [0.6766, 0.857] 
+2022-05-20 03:23:18.065351: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:23:18.470927: lr: 4e-06 
+2022-05-20 03:23:18.471126: saving scheduled checkpoint file... 
+2022-05-20 03:23:18.529032: saving checkpoint... 
+2022-05-20 03:23:19.331541: done, saving took 0.86 seconds 
+2022-05-20 03:23:19.340604: done 
+2022-05-20 03:23:38.261772: This epoch took 509.360708 s
+ 
+2022-05-20 03:23:38.262043: 
+epoch:  117 
+2022-05-20 03:31:21.822348: train loss : 0.0949 
+2022-05-20 03:31:48.177223: validation loss: 0.1081 
+2022-05-20 03:31:48.177839: Average global foreground Dice: [0.6756, 0.8533] 
+2022-05-20 03:31:48.177922: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:31:48.591123: lr: 3e-06 
+2022-05-20 03:31:48.591320: saving scheduled checkpoint file... 
+2022-05-20 03:31:48.649183: saving checkpoint... 
+2022-05-20 03:31:49.587246: done, saving took 1.00 seconds 
+2022-05-20 03:31:49.596138: done 
+2022-05-20 03:32:46.489815: This epoch took 548.227656 s
+ 
+2022-05-20 03:32:46.490309: 
+epoch:  118 
+2022-05-20 03:40:29.391732: train loss : 0.0937 
+2022-05-20 03:40:55.720541: validation loss: 0.1077 
+2022-05-20 03:40:55.721191: Average global foreground Dice: [0.6845, 0.8654] 
+2022-05-20 03:40:55.721326: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:40:56.139208: lr: 2e-06 
+2022-05-20 03:40:56.139404: saving scheduled checkpoint file... 
+2022-05-20 03:40:56.197829: saving checkpoint... 
+2022-05-20 03:40:57.126968: done, saving took 0.99 seconds 
+2022-05-20 03:40:57.135924: done 
+2022-05-20 03:41:15.609170: This epoch took 509.118477 s
+ 
+2022-05-20 03:41:15.609459: 
+epoch:  119 
+2022-05-20 03:48:58.805049: train loss : 0.0983 
+2022-05-20 03:49:25.164801: validation loss: 0.1256 
+2022-05-20 03:49:25.165486: Average global foreground Dice: [0.6756, 0.8645] 
+2022-05-20 03:49:25.165570: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:49:25.569607: lr: 1e-06 
+2022-05-20 03:49:25.569806: saving scheduled checkpoint file... 
+2022-05-20 03:49:25.627492: saving checkpoint... 
+2022-05-20 03:49:26.562666: done, saving took 0.99 seconds 
+2022-05-20 03:49:26.572377: done 
+2022-05-20 03:49:52.132060: This epoch took 516.522492 s
+ 
+2022-05-20 03:49:52.132335: 
+epoch:  120 
+2022-05-20 03:57:35.176126: train loss : 0.0999 
+2022-05-20 03:58:01.531392: validation loss: 0.1005 
+2022-05-20 03:58:01.532017: Average global foreground Dice: [0.6722, 0.8625] 
+2022-05-20 03:58:01.532104: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:58:01.950476: lr: 1e-06 
+2022-05-20 03:58:01.950664: saving scheduled checkpoint file... 
+2022-05-20 03:58:02.008911: saving checkpoint... 
+2022-05-20 03:58:02.898809: done, saving took 0.95 seconds 
+2022-05-20 03:58:02.908306: done 
+2022-05-20 03:58:28.736899: This epoch took 516.604453 s
+ 
+2022-05-20 03:58:28.737174: 
+epoch:  121 
+2022-05-20 04:06:11.718487: train loss : 0.0949 
+2022-05-20 04:06:38.102492: validation loss: 0.1138 
+2022-05-20 04:06:38.103064: Average global foreground Dice: [0.6876, 0.851] 
+2022-05-20 04:06:38.103145: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 04:06:38.513355: lr: 1e-06 
+2022-05-20 04:06:38.513543: saving scheduled checkpoint file... 
+2022-05-20 04:06:38.571612: saving checkpoint... 
+2022-05-20 04:06:39.500064: done, saving took 0.99 seconds 
+2022-05-20 04:06:39.509925: done 
+2022-05-20 04:07:06.909122: This epoch took 518.171830 s
+ 
+2022-05-20 04:07:06.909507: 
+epoch:  122 
+2022-05-20 04:14:49.729285: train loss : 0.0974 
+2022-05-20 04:15:16.127015: validation loss: 0.1199 
+2022-05-20 04:15:16.127755: Average global foreground Dice: [0.665, 0.867] 
+2022-05-20 04:15:16.127844: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 04:15:16.534158: lr: 0.0 
+2022-05-20 04:15:16.534334: saving scheduled checkpoint file... 
+2022-05-20 04:15:16.591745: saving checkpoint... 
+2022-05-20 04:15:17.451854: done, saving took 0.92 seconds 
+2022-05-20 04:15:17.460165: done 
+2022-05-20 04:15:39.474860: This epoch took 512.565244 s
+ 
+2022-05-20 04:15:39.475165: 
+epoch:  123 
+2022-05-20 04:23:22.843241: train loss : 0.0932 
+2022-05-20 04:23:49.158400: validation loss: 0.0986 
+2022-05-20 04:23:49.159024: Average global foreground Dice: [0.6783, 0.8611] 
+2022-05-20 04:23:49.159109: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 04:23:49.565722: lr: 0.0 
+2022-05-20 04:23:49.565911: saving scheduled checkpoint file... 
+2022-05-20 04:23:49.625033: saving checkpoint... 
+2022-05-20 04:23:50.571559: done, saving took 1.01 seconds 
+2022-05-20 04:23:50.581537: done 
+2022-05-20 04:24:08.915749: This epoch took 509.440476 s
+ 
+2022-05-20 04:24:08.916007: 
+epoch:  124 
+2022-05-20 04:31:52.328473: train loss : 0.0927 
+2022-05-20 04:32:18.650634: validation loss: 0.0996 
+2022-05-20 04:32:18.652027: Average global foreground Dice: [0.6767, 0.8734] 
+2022-05-20 04:32:18.652123: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 04:32:19.064733: lr: 0.0 
+2022-05-20 04:32:19.064918: saving scheduled checkpoint file... 
+2022-05-20 04:32:19.121712: saving checkpoint... 
+2022-05-20 04:32:20.058147: done, saving took 0.99 seconds 
+2022-05-20 04:32:20.068127: done 
+2022-05-20 04:32:44.013047: This epoch took 515.096796 s
+ 
+2022-05-20 04:32:44.069348: saving checkpoint... 
+2022-05-20 04:32:44.745355: done, saving took 0.73 seconds 

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/debug.json

@@ -0,0 +1,106 @@
+{
+    "all_tr_losses": "[0.3387577, 0.22670184, 0.20843363, 0.18841243, 0.17535995, 0.16141209, 0.15405571, 0.15061752, 0.15452078, 0.15204631, 0.13644604, 0.15092108, 0.14956972, 0.15689449, 0.1389342, 0.15144578, 0.14459546, 0.14335355, 0.13697049, 0.13083953, 0.13117258, 0.13288935, 0.121476956, 0.13318454, 0.13405542, 0.12568034, 0.12733866, 0.12671013, 0.12438219, 0.12844828, 0.12240422, 0.123266265, 0.1184496, 0.13073105, 0.1177899, 0.12049695, 0.11747541, 0.11702523, 0.123542555, 0.123520136, 0.115563944, 0.12850183, 0.11967583, 0.10957871, 0.1181682, 0.1152821, 0.11642279, 0.113893405, 0.119086206, 0.11678498, 0.123117104, 0.115178846, 0.11567018, 0.11985725, 0.1193085, 0.11845499, 0.11243107, 0.10831526, 0.10665122, 0.11258734, 0.115851745, 0.103712484, 0.11779399, 0.112267956, 0.113086045, 0.105029725, 0.09838733, 0.107105955, 0.10617215, 0.104107745, 0.10817049]",
+    "all_val_eval_metrics": "[0.233077535801157, 0.4480676378630247, 0.4404973365469221, 0.5027243923707809, 0.5741376998387168, 0.5867073937865842, 0.6321437400284091, 0.4927105040099135, 0.6383731827011241, 0.6758598205366171, 0.6528591895183398, 0.6637023021051314, 0.6314350073658983, 0.6958482818557177, 0.6989928391929008, 0.6912398961084707, 0.6056596686051938, 0.7117652265461507, 0.7327134497429773, 0.7178654087577003, 0.7049232178373495, 0.7322875032534468, 0.7283864767120573, 0.703206818910838, 0.6927400603591269, 0.6898464873178634, 0.7178142991927289, 0.7232832695864841, 0.7031412997778539, 0.7459311406337356, 0.7169932042971585, 0.6961739934424764, 0.7405176514536038, 0.7500159841744369, 0.7223372438887824, 0.7509090590750573, 0.73228006616551, 0.690102673090841, 0.7264675974757193, 0.7339441598712162, 0.734639125446374, 0.747450182941839, 0.7020488500687379, 0.7402710892243097, 0.7477285325027372, 0.7586862586951484, 0.7242972876877314, 0.7194051296686022, 0.7512135982956042, 0.753453557303387, 0.7631378971540221, 0.7503152041647418, 0.7545195706241937, 0.7526524973870545, 0.7577355105991712, 0.7543985646253323, 0.7504404477878608, 0.7558896428716659, 0.7615744371323204, 0.7533564094350783, 0.7656408155315066, 0.7363721082843939, 0.752561029693577, 0.7385154899854834, 0.7577475393231858, 0.7401747061930488, 0.7604726162422142, 0.7553712579261274, 0.7234086586128576, 0.7414167035104149, 0.7605660772328076]",
+    "all_val_losses": "[0.2742561, 0.24531353, 0.18815613, 0.17646404, 0.15832248, 0.2023902, 0.15069067, 0.16915901, 0.15229218, 0.14243098, 0.14826919, 0.15193358, 0.14406703, 0.14335576, 0.1447152, 0.13216203, 0.16123854, 0.1316906, 0.11502855, 0.11714242, 0.13802771, 0.13044147, 0.123741314, 0.1385207, 0.117034934, 0.14418751, 0.12603754, 0.12535158, 0.13195664, 0.1259039, 0.12737633, 0.13186885, 0.1261872, 0.12275005, 0.14040247, 0.13383172, 0.12274504, 0.14137703, 0.1274172, 0.118551254, 0.123299085, 0.13093303, 0.12997475, 0.10903824, 0.11815379, 0.12656236, 0.12919399, 0.12636602, 0.14482778, 0.11013544, 0.10813198, 0.1311769, 0.12588133, 0.12132832, 0.114321955, 0.1252415, 0.12425144, 0.108929895, 0.118857205, 0.11405925, 0.119004786, 0.119694024, 0.10662134, 0.13051103, 0.12756462, 0.11987198, 0.12853155, 0.12863775, 0.1274451, 0.12693381, 0.12606291]",
+    "all_val_losses_tr_mode": "[]",
+    "also_val_in_tr_mode": "False",
+    "amp_grad_scaler": "<torch.cuda.amp.grad_scaler.GradScaler object at 0x7f64f1719240>",
+    "args": "Namespace(batch_size=2, config='./configs/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32.yaml', continue_training=False, crop_size=[64, 192, 192], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='1', fp32=False, hdfs_base='GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=125, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task008_HepaticVessel', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)",
+    "base_num_features": "32",
+    "basic_generator_patch_size": "[178 308 233]",
+    "batch_dice": "True",
+    "batch_size": "2",
+    "best_MA_tr_loss_for_patience": "None",
+    "best_epoch_based_on_MA_tr_loss": "None",
+    "best_val_eval_criterion_MA": "None",
+    "classes": "[1, 2]",
+    "conv_per_stage": "2",
+    "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-0.5235987755982988, 0.5235987755982988), 'rotation_y': (-0.5235987755982988, 0.5235987755982988), 'rotation_z': (-0.5235987755982988, 0.5235987755982988), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([ 64, 192, 192])}",
+    "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel",
+    "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]",
+    "deterministic": "False",
+    "disable_ds": "True",
+    "distribute_batch_size": "False",
+    "dl_tr": "<dataset_loading.DataLoader3D object at 0x7f654331bdd8>",
+    "dl_val": "<dataset_loading.DataLoader3D object at 0x7f654c9431d0>",
+    "do_dummy_2D_aug": "False",
+    "ds_loss_weights": "[1. 0. 0. 0. 0.]",
+    "epoch": "71",
+    "experiment_name": "nnUNetTrainerV2_DDP",
+    "find_zero_weight_decay": "False",
+    "fold": "1",
+    "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetData_plans_v2.1_stage1",
+    "fp16": "True",
+    "global_batch_size": "16",
+    "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/gt_segmentations",
+    "inference_pad_border_mode": "constant",
+    "inference_pad_kwargs": "{'constant_values': 0}",
+    "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl', 1, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', '/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel', True, 1, True, False, False, True)",
+    "initial_lr": "0.0003",
+    "input_size": "[64, 192, 192]",
+    "is_spatial_aug_only": "False",
+    "is_ssl": "False",
+    "layer_decay": "1.0",
+    "local_rank": "0",
+    "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/training_log_2022_4_12_01_47_45.txt",
+    "loss": "None",
+    "lr_scheduler": "<optimizers.lr_scheduler.LinearWarmupCosineAnnealingLR object at 0x7f64f1719b38>",
+    "lr_scheduler_eps": "0.001",
+    "lr_scheduler_name": "warmup_cosine",
+    "lr_scheduler_patience": "30",
+    "lr_threshold": "1e-06",
+    "max_num_epochs": "125",
+    "min_lr": "0",
+    "min_region_size_per_class": "None",
+    "min_size_per_class": "None",
+    "model": "Generic_TransUNet_max_ppbp",
+    "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}",
+    "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]",
+    "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]]",
+    "net_pool_per_axis": "[4, 5, 5]",
+    "normalization_schemes": "OrderedDict([(0, 'CT')])",
+    "num_batches_per_epoch": "250",
+    "num_classes": "3",
+    "num_input_channels": "1",
+    "num_val_batches_per_epoch": "50",
+    "online_eval_fn": "[]",
+    "online_eval_foreground_dc": "[]",
+    "online_eval_fp": "[]",
+    "online_eval_tp": "[]",
+    "only_keep_largest_connected_component": "None",
+    "optim_name": "adamw",
+    "optimizer": "AdamW (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    eps: 1e-08\n    initial_lr: 0.0003\n    lr: 0.00013567978905008846\n    weight_decay: 3e-05\n)",
+    "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1",
+    "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32",
+    "oversample_foreground_percent": "0.0",
+    "pad_all_sides": "None",
+    "patch_size": "[ 64 192 192]",
+    "patience": "50",
+    "pin_memory": "True",
+    "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl",
+    "reclip": "None",
+    "regions_class_order": "None",
+    "resume": "auto",
+    "save_best_checkpoint": "True",
+    "save_every": "1",
+    "save_final_checkpoint": "True",
+    "save_intermediate_checkpoints": "True",
+    "save_latest_only": "True",
+    "stage": "1",
+    "threeD": "True",
+    "tr_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7f654331fba8>",
+    "train_loss_MA": "None",
+    "train_loss_MA_alpha": "0.93",
+    "train_loss_MA_eps": "0.0005",
+    "transpose_backward": "[0, 1, 2]",
+    "transpose_forward": "[0, 1, 2]",
+    "unpack_data": "True",
+    "use_mask_for_norm": "OrderedDict([(0, False)])",
+    "use_progress_bar": "False",
+    "val_eval_criterion_MA": "None",
+    "val_eval_criterion_alpha": "0.9",
+    "val_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7f65430aa048>",
+    "warmup_epochs": "10",
+    "was_initialized": "True",
+    "weight_decay": "3e-05"
+}

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_best.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8234e587470532959eed0bd70ede576a7799b69eb81123be03550133bd5581f0
+size 496529338

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_best.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2097f69bb201473667d1842c13e15b59d686b7e6db4c94f0121336d326b6c24
+size 188314

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_final_checkpoint.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:399f0625d42c8894614eb3bf5a03fdd14f3426ecf1bb509d38642d6bef46b528
+size 496529530

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_final_checkpoint.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2097f69bb201473667d1842c13e15b59d686b7e6db4c94f0121336d326b6c24
+size 188314

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_latest.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad2d3433793c557c8be722845c79ac21dc90b3061ffce24e9cda8262923fb819
+size 496529530

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_latest.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2097f69bb201473667d1842c13e15b59d686b7e6db4c94f0121336d326b6c24
+size 188314

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/training_log_2022_4_12_01_47_45.txt

@@ -0,0 +1,771 @@
+Starting... 
+2022-04-12 01:47:45.715590: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/splits_final.pkl 
+2022-04-12 01:47:45.716197: The split file contains 5 splits. 
+2022-04-12 01:47:45.716254: Desired fold for training: 1 
+2022-04-12 01:47:45.716299: This split has 242 training and 61 validation cases. 
+2022-04-12 01:49:23.672634: TRAINING KEYS:
+ odict_keys(['hepaticvessel_001', 'hepaticvessel_002', 'hepaticvessel_004', 'hepaticvessel_005', 'hepaticvessel_007', 'hepaticvessel_008', 'hepaticvessel_010', 'hepaticvessel_011', 'hepaticvessel_013', 'hepaticvessel_018', 'hepaticvessel_019', 'hepaticvessel_020', 'hepaticvessel_022', 'hepaticvessel_026', 'hepaticvessel_028', 'hepaticvessel_030', 'hepaticvessel_031', 'hepaticvessel_032', 'hepaticvessel_033', 'hepaticvessel_039', 'hepaticvessel_040', 'hepaticvessel_042', 'hepaticvessel_046', 'hepaticvessel_049', 'hepaticvessel_050', 'hepaticvessel_051', 'hepaticvessel_052', 'hepaticvessel_053', 'hepaticvessel_057', 'hepaticvessel_058', 'hepaticvessel_059', 'hepaticvessel_061', 'hepaticvessel_066', 'hepaticvessel_067', 'hepaticvessel_068', 'hepaticvessel_070', 'hepaticvessel_072', 'hepaticvessel_075', 'hepaticvessel_077', 'hepaticvessel_078', 'hepaticvessel_080', 'hepaticvessel_081', 'hepaticvessel_082', 'hepaticvessel_084', 'hepaticvessel_085', 'hepaticvessel_086', 'hepaticvessel_087', 'hepaticvessel_088', 'hepaticvessel_089', 'hepaticvessel_090', 'hepaticvessel_091', 'hepaticvessel_092', 'hepaticvessel_093', 'hepaticvessel_094', 'hepaticvessel_096', 'hepaticvessel_097', 'hepaticvessel_098', 'hepaticvessel_100', 'hepaticvessel_102', 'hepaticvessel_108', 'hepaticvessel_110', 'hepaticvessel_111', 'hepaticvessel_112', 'hepaticvessel_113', 'hepaticvessel_115', 'hepaticvessel_116', 'hepaticvessel_117', 'hepaticvessel_119', 'hepaticvessel_121', 'hepaticvessel_123', 'hepaticvessel_124', 'hepaticvessel_125', 'hepaticvessel_127', 'hepaticvessel_128', 'hepaticvessel_129', 'hepaticvessel_133', 'hepaticvessel_136', 'hepaticvessel_137', 'hepaticvessel_138', 'hepaticvessel_139', 'hepaticvessel_140', 'hepaticvessel_141', 'hepaticvessel_144', 'hepaticvessel_146', 'hepaticvessel_147', 'hepaticvessel_148', 'hepaticvessel_150', 'hepaticvessel_151', 'hepaticvessel_154', 'hepaticvessel_157', 'hepaticvessel_158', 'hepaticvessel_159', 'hepaticvessel_160', 'hepaticvessel_161', 'hepaticvessel_163', 'hepaticvessel_164', 'hepaticvessel_166', 'hepaticvessel_167', 'hepaticvessel_171', 'hepaticvessel_172', 'hepaticvessel_173', 'hepaticvessel_174', 'hepaticvessel_175', 'hepaticvessel_177', 'hepaticvessel_178', 'hepaticvessel_179', 'hepaticvessel_181', 'hepaticvessel_183', 'hepaticvessel_184', 'hepaticvessel_189', 'hepaticvessel_193', 'hepaticvessel_194', 'hepaticvessel_195', 'hepaticvessel_197', 'hepaticvessel_198', 'hepaticvessel_199', 'hepaticvessel_200', 'hepaticvessel_201', 'hepaticvessel_203', 'hepaticvessel_206', 'hepaticvessel_207', 'hepaticvessel_208', 'hepaticvessel_209', 'hepaticvessel_211', 'hepaticvessel_213', 'hepaticvessel_214', 'hepaticvessel_215', 'hepaticvessel_217', 'hepaticvessel_218', 'hepaticvessel_221', 'hepaticvessel_222', 'hepaticvessel_223', 'hepaticvessel_224', 'hepaticvessel_227', 'hepaticvessel_229', 'hepaticvessel_233', 'hepaticvessel_234', 'hepaticvessel_236', 'hepaticvessel_237', 'hepaticvessel_238', 'hepaticvessel_244', 'hepaticvessel_245', 'hepaticvessel_246', 'hepaticvessel_248', 'hepaticvessel_259', 'hepaticvessel_262', 'hepaticvessel_263', 'hepaticvessel_265', 'hepaticvessel_266', 'hepaticvessel_268', 'hepaticvessel_269', 'hepaticvessel_270', 'hepaticvessel_271', 'hepaticvessel_272', 'hepaticvessel_274', 'hepaticvessel_275', 'hepaticvessel_278', 'hepaticvessel_279', 'hepaticvessel_280', 'hepaticvessel_281', 'hepaticvessel_282', 'hepaticvessel_284', 'hepaticvessel_286', 'hepaticvessel_289', 'hepaticvessel_290', 'hepaticvessel_291', 'hepaticvessel_293', 'hepaticvessel_294', 'hepaticvessel_296', 'hepaticvessel_300', 'hepaticvessel_305', 'hepaticvessel_307', 'hepaticvessel_308', 'hepaticvessel_309', 'hepaticvessel_314', 'hepaticvessel_318', 'hepaticvessel_319', 'hepaticvessel_320', 'hepaticvessel_322', 'hepaticvessel_323', 'hepaticvessel_325', 'hepaticvessel_327', 'hepaticvessel_329', 'hepaticvessel_333', 'hepaticvessel_335', 'hepaticvessel_336', 'hepaticvessel_338', 'hepaticvessel_344', 'hepaticvessel_350', 'hepaticvessel_353', 'hepaticvessel_358', 'hepaticvessel_359', 'hepaticvessel_361', 'hepaticvessel_363', 'hepaticvessel_367', 'hepaticvessel_368', 'hepaticvessel_369', 'hepaticvessel_371', 'hepaticvessel_372', 'hepaticvessel_373', 'hepaticvessel_374', 'hepaticvessel_375', 'hepaticvessel_378', 'hepaticvessel_379', 'hepaticvessel_382', 'hepaticvessel_385', 'hepaticvessel_386', 'hepaticvessel_393', 'hepaticvessel_396', 'hepaticvessel_397', 'hepaticvessel_398', 'hepaticvessel_400', 'hepaticvessel_401', 'hepaticvessel_402', 'hepaticvessel_404', 'hepaticvessel_411', 'hepaticvessel_412', 'hepaticvessel_416', 'hepaticvessel_419', 'hepaticvessel_420', 'hepaticvessel_422', 'hepaticvessel_423', 'hepaticvessel_424', 'hepaticvessel_425', 'hepaticvessel_427', 'hepaticvessel_429', 'hepaticvessel_431', 'hepaticvessel_432', 'hepaticvessel_433', 'hepaticvessel_434', 'hepaticvessel_437', 'hepaticvessel_438', 'hepaticvessel_441', 'hepaticvessel_442', 'hepaticvessel_444', 'hepaticvessel_445', 'hepaticvessel_446', 'hepaticvessel_447', 'hepaticvessel_453', 'hepaticvessel_454', 'hepaticvessel_455', 'hepaticvessel_458']) 
+2022-04-12 01:49:23.672823: VALIDATION KEYS:
+ odict_keys(['hepaticvessel_016', 'hepaticvessel_023', 'hepaticvessel_025', 'hepaticvessel_027', 'hepaticvessel_029', 'hepaticvessel_044', 'hepaticvessel_062', 'hepaticvessel_065', 'hepaticvessel_071', 'hepaticvessel_079', 'hepaticvessel_083', 'hepaticvessel_103', 'hepaticvessel_104', 'hepaticvessel_131', 'hepaticvessel_132', 'hepaticvessel_142', 'hepaticvessel_145', 'hepaticvessel_165', 'hepaticvessel_185', 'hepaticvessel_186', 'hepaticvessel_192', 'hepaticvessel_196', 'hepaticvessel_210', 'hepaticvessel_219', 'hepaticvessel_225', 'hepaticvessel_230', 'hepaticvessel_231', 'hepaticvessel_235', 'hepaticvessel_240', 'hepaticvessel_242', 'hepaticvessel_255', 'hepaticvessel_256', 'hepaticvessel_258', 'hepaticvessel_285', 'hepaticvessel_287', 'hepaticvessel_297', 'hepaticvessel_299', 'hepaticvessel_303', 'hepaticvessel_321', 'hepaticvessel_324', 'hepaticvessel_330', 'hepaticvessel_332', 'hepaticvessel_340', 'hepaticvessel_341', 'hepaticvessel_345', 'hepaticvessel_348', 'hepaticvessel_349', 'hepaticvessel_356', 'hepaticvessel_362', 'hepaticvessel_383', 'hepaticvessel_384', 'hepaticvessel_388', 'hepaticvessel_389', 'hepaticvessel_391', 'hepaticvessel_406', 'hepaticvessel_407', 'hepaticvessel_408', 'hepaticvessel_409', 'hepaticvessel_440', 'hepaticvessel_443', 'hepaticvessel_456']) 
+2022-04-12 01:49:37.964392: loading checkpoint /opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_1/model_latest.model train= True 
+2022-04-12 01:49:38.804048: lr: 0.000132 
+2022-04-12 01:49:53.151024: 
+epoch:  71 
+2022-04-12 01:57:49.694312: train loss : 0.1032 
+2022-04-12 01:58:16.156663: validation loss: 0.1305 
+2022-04-12 01:58:16.157230: Average global foreground Dice: [0.643, 0.8416] 
+2022-04-12 01:58:16.157407: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 01:58:16.571024: lr: 0.000132 
+2022-04-12 01:58:16.571239: saving scheduled checkpoint file... 
+2022-04-12 01:58:16.631706: saving checkpoint... 
+2022-04-12 01:58:17.447424: done, saving took 0.88 seconds 
+2022-04-12 01:58:17.454373: done 
+2022-04-12 01:58:47.022275: This epoch took 533.870604 s
+ 
+2022-04-12 01:58:47.022511: 
+epoch:  72 
+2022-04-12 02:06:29.648321: train loss : 0.1011 
+2022-04-12 02:06:55.986377: validation loss: 0.1333 
+2022-04-12 02:06:55.987037: Average global foreground Dice: [0.6506, 0.8528] 
+2022-04-12 02:06:55.987121: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 02:06:56.392863: lr: 0.000128 
+2022-04-12 02:06:56.393059: saving scheduled checkpoint file... 
+2022-04-12 02:06:56.443024: saving checkpoint... 
+2022-04-12 02:06:57.321468: done, saving took 0.93 seconds 
+2022-04-12 02:06:57.328844: done 
+2022-04-12 02:06:57.356558: saving checkpoint... 
+2022-04-12 02:06:57.917594: done, saving took 0.59 seconds 
+2022-04-12 02:07:23.173922: This epoch took 516.151274 s
+ 
+2022-04-12 02:07:23.174200: 
+epoch:  73 
+2022-04-12 02:15:06.193495: train loss : 0.1096 
+2022-04-12 02:15:32.526185: validation loss: 0.1035 
+2022-04-12 02:15:32.526824: Average global foreground Dice: [0.6583, 0.8722] 
+2022-04-12 02:15:32.526909: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 02:15:32.995968: lr: 0.000124 
+2022-04-12 02:15:32.996156: saving scheduled checkpoint file... 
+2022-04-12 02:15:33.046960: saving checkpoint... 
+2022-04-12 02:15:33.980390: done, saving took 0.98 seconds 
+2022-04-12 02:15:33.988710: done 
+2022-04-12 02:15:34.017741: saving checkpoint... 
+2022-04-12 02:15:34.889052: done, saving took 0.90 seconds 
+2022-04-12 02:16:02.025848: This epoch took 518.851540 s
+ 
+2022-04-12 02:16:02.026114: 
+epoch:  74 
+2022-04-12 02:23:44.390225: train loss : 0.1066 
+2022-04-12 02:24:10.791477: validation loss: 0.1189 
+2022-04-12 02:24:10.792030: Average global foreground Dice: [0.6459, 0.8639] 
+2022-04-12 02:24:10.792114: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 02:24:11.200119: lr: 0.000119 
+2022-04-12 02:24:11.200323: saving scheduled checkpoint file... 
+2022-04-12 02:24:11.250188: saving checkpoint... 
+2022-04-12 02:24:12.043696: done, saving took 0.84 seconds 
+2022-04-12 02:24:12.050386: done 
+2022-04-12 02:24:12.076408: saving checkpoint... 
+2022-04-12 02:24:12.850010: done, saving took 0.80 seconds 
+2022-04-12 02:24:35.732872: This epoch took 513.706657 s
+ 
+2022-04-12 02:24:35.733162: 
+epoch:  75 
+2022-04-12 02:32:18.218835: train loss : 0.1074 
+2022-04-12 02:32:44.560797: validation loss: 0.1117 
+2022-04-12 02:32:44.561381: Average global foreground Dice: [0.6595, 0.8315] 
+2022-04-12 02:32:44.561466: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 02:32:44.964844: lr: 0.000115 
+2022-04-12 02:32:44.965041: saving scheduled checkpoint file... 
+2022-04-12 02:32:45.014532: saving checkpoint... 
+2022-04-12 02:32:45.916269: done, saving took 0.95 seconds 
+2022-04-12 02:32:45.923709: done 
+2022-04-12 02:33:08.392709: This epoch took 512.659441 s
+ 
+2022-04-12 02:33:08.393205: 
+epoch:  76 
+2022-04-12 02:40:50.204226: train loss : 0.1033 
+2022-04-12 02:41:16.547214: validation loss: 0.1284 
+2022-04-12 02:41:16.547741: Average global foreground Dice: [0.6446, 0.8595] 
+2022-04-12 02:41:16.547825: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 02:41:16.955729: lr: 0.000112 
+2022-04-12 02:41:16.955919: saving scheduled checkpoint file... 
+2022-04-12 02:41:17.005730: saving checkpoint... 
+2022-04-12 02:41:17.905257: done, saving took 0.95 seconds 
+2022-04-12 02:41:17.913049: done 
+2022-04-12 02:41:17.938973: saving checkpoint... 
+2022-04-12 02:41:18.816704: done, saving took 0.90 seconds 
+2022-04-12 02:41:47.085027: This epoch took 518.691720 s
+ 
+2022-04-12 02:41:47.085310: 
+epoch:  77 
+2022-04-12 02:49:29.633742: train loss : 0.0987 
+2022-04-12 02:49:56.015526: validation loss: 0.1085 
+2022-04-12 02:49:56.016048: Average global foreground Dice: [0.661, 0.8382] 
+2022-04-12 02:49:56.016127: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 02:49:56.423056: lr: 0.000108 
+2022-04-12 02:49:56.423248: saving scheduled checkpoint file... 
+2022-04-12 02:49:56.472756: saving checkpoint... 
+2022-04-12 02:49:57.363767: done, saving took 0.94 seconds 
+2022-04-12 02:49:57.371142: done 
+2022-04-12 02:49:57.397474: saving checkpoint... 
+2022-04-12 02:49:58.250980: done, saving took 0.88 seconds 
+2022-04-12 02:50:17.277142: This epoch took 510.191730 s
+ 
+2022-04-12 02:50:17.277422: 
+epoch:  78 
+2022-04-12 02:58:00.183569: train loss : 0.1005 
+2022-04-12 02:58:26.547135: validation loss: 0.1207 
+2022-04-12 02:58:26.547724: Average global foreground Dice: [0.642, 0.8506] 
+2022-04-12 02:58:26.547811: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 02:58:26.953531: lr: 0.000104 
+2022-04-12 02:58:26.953735: saving scheduled checkpoint file... 
+2022-04-12 02:58:27.003320: saving checkpoint... 
+2022-04-12 02:58:27.795536: done, saving took 0.84 seconds 
+2022-04-12 02:58:27.802729: done 
+2022-04-12 02:58:47.164164: This epoch took 509.886633 s
+ 
+2022-04-12 02:58:47.164483: 
+epoch:  79 
+2022-04-12 03:06:29.668583: train loss : 0.1029 
+2022-04-12 03:06:56.021065: validation loss: 0.1178 
+2022-04-12 03:06:56.021674: Average global foreground Dice: [0.6534, 0.8324] 
+2022-04-12 03:06:56.021758: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 03:06:56.429858: lr: 0.0001 
+2022-04-12 03:06:56.430068: saving scheduled checkpoint file... 
+2022-04-12 03:06:56.480796: saving checkpoint... 
+2022-04-12 03:06:57.429707: done, saving took 1.00 seconds 
+2022-04-12 03:06:57.437105: done 
+2022-04-12 03:07:19.194026: This epoch took 512.029446 s
+ 
+2022-04-12 03:07:19.194331: 
+epoch:  80 
+2022-04-12 03:15:01.648175: train loss : 0.1022 
+2022-04-12 03:15:28.021894: validation loss: 0.1077 
+2022-04-12 03:15:28.022485: Average global foreground Dice: [0.6583, 0.8729] 
+2022-04-12 03:15:28.022582: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 03:15:28.430223: lr: 9.6e-05 
+2022-04-12 03:15:28.430423: saving scheduled checkpoint file... 
+2022-04-12 03:15:28.479786: saving checkpoint... 
+2022-04-12 03:15:29.378370: done, saving took 0.95 seconds 
+2022-04-12 03:15:29.385749: done 
+2022-04-12 03:15:29.411012: saving checkpoint... 
+2022-04-12 03:15:30.287992: done, saving took 0.90 seconds 
+2022-04-12 03:15:55.006488: This epoch took 515.812053 s
+ 
+2022-04-12 03:15:55.006775: 
+epoch:  81 
+2022-04-12 03:23:37.473773: train loss : 0.0935 
+2022-04-12 03:24:03.849096: validation loss: 0.1134 
+2022-04-12 03:24:03.849644: Average global foreground Dice: [0.648, 0.7903] 
+2022-04-12 03:24:03.849741: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 03:24:04.256096: lr: 9.2e-05 
+2022-04-12 03:24:04.256285: saving scheduled checkpoint file... 
+2022-04-12 03:24:04.305759: saving checkpoint... 
+2022-04-12 03:24:05.082351: done, saving took 0.83 seconds 
+2022-04-12 03:24:05.089416: done 
+2022-04-12 03:24:49.673854: This epoch took 534.666977 s
+ 
+2022-04-12 03:24:49.674152: 
+epoch:  82 
+2022-04-12 03:32:32.153378: train loss : 0.0948 
+2022-04-12 03:32:58.510216: validation loss: 0.1180 
+2022-04-12 03:32:58.510844: Average global foreground Dice: [0.6668, 0.8711] 
+2022-04-12 03:32:58.510931: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 03:32:58.911346: lr: 8.8e-05 
+2022-04-12 03:32:58.911533: saving scheduled checkpoint file... 
+2022-04-12 03:32:58.963757: saving checkpoint... 
+2022-04-12 03:32:59.879307: done, saving took 0.97 seconds 
+2022-04-12 03:32:59.886877: done 
+2022-04-12 03:33:20.449904: This epoch took 510.775651 s
+ 
+2022-04-12 03:33:20.450195: 
+epoch:  83 
+2022-04-12 03:41:02.915692: train loss : 0.1014 
+2022-04-12 03:41:29.292401: validation loss: 0.1061 
+2022-04-12 03:41:29.293012: Average global foreground Dice: [0.6479, 0.8475] 
+2022-04-12 03:41:29.293097: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 03:41:29.691291: lr: 8.5e-05 
+2022-04-12 03:41:29.691468: saving scheduled checkpoint file... 
+2022-04-12 03:41:29.740707: saving checkpoint... 
+2022-04-12 03:41:30.624274: done, saving took 0.93 seconds 
+2022-04-12 03:41:30.632542: done 
+2022-04-12 03:41:50.862592: This epoch took 510.412282 s
+ 
+2022-04-12 03:41:50.862871: 
+epoch:  84 
+2022-04-12 03:49:33.934120: train loss : 0.1059 
+2022-04-12 03:50:00.304400: validation loss: 0.1166 
+2022-04-12 03:50:00.304986: Average global foreground Dice: [0.6326, 0.8662] 
+2022-04-12 03:50:00.305071: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 03:50:00.776315: lr: 8.1e-05 
+2022-04-12 03:50:00.776498: saving scheduled checkpoint file... 
+2022-04-12 03:50:00.825917: saving checkpoint... 
+2022-04-12 03:50:01.626990: done, saving took 0.85 seconds 
+2022-04-12 03:50:01.657367: done 
+2022-04-12 03:50:39.177516: This epoch took 528.314546 s
+ 
+2022-04-12 03:50:39.177794: 
+epoch:  85 
+2022-04-12 03:58:21.691803: train loss : 0.0956 
+2022-04-12 03:58:48.036259: validation loss: 0.1199 
+2022-04-12 03:58:48.036843: Average global foreground Dice: [0.6538, 0.8629] 
+2022-04-12 03:58:48.036927: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 03:58:48.438333: lr: 7.7e-05 
+2022-04-12 03:58:48.438528: saving scheduled checkpoint file... 
+2022-04-12 03:58:48.488003: saving checkpoint... 
+2022-04-12 03:58:49.389500: done, saving took 0.95 seconds 
+2022-04-12 03:58:49.397269: done 
+2022-04-12 03:58:49.433043: saving checkpoint... 
+2022-04-12 03:58:50.288560: done, saving took 0.89 seconds 
+2022-04-12 03:59:11.186197: This epoch took 512.008299 s
+ 
+2022-04-12 03:59:11.186495: 
+epoch:  86 
+2022-04-12 04:06:53.730433: train loss : 0.1128 
+2022-04-12 04:07:20.065057: validation loss: 0.1089 
+2022-04-12 04:07:20.065630: Average global foreground Dice: [0.6651, 0.871] 
+2022-04-12 04:07:20.065715: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 04:07:20.475675: lr: 7.4e-05 
+2022-04-12 04:07:20.475862: saving scheduled checkpoint file... 
+2022-04-12 04:07:20.525327: saving checkpoint... 
+2022-04-12 04:07:21.429981: done, saving took 0.95 seconds 
+2022-04-12 04:07:21.437414: done 
+2022-04-12 04:07:21.463322: saving checkpoint... 
+2022-04-12 04:07:22.345131: done, saving took 0.91 seconds 
+2022-04-12 04:08:12.194013: This epoch took 541.007360 s
+ 
+2022-04-12 04:08:12.194272: 
+epoch:  87 
+2022-04-12 04:15:55.084049: train loss : 0.1040 
+2022-04-12 04:16:21.452374: validation loss: 0.1230 
+2022-04-12 04:16:21.453017: Average global foreground Dice: [0.6493, 0.8623] 
+2022-04-12 04:16:21.453100: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 04:16:21.854911: lr: 7e-05 
+2022-04-12 04:16:21.855108: saving scheduled checkpoint file... 
+2022-04-12 04:16:21.905278: saving checkpoint... 
+2022-04-12 04:16:22.659439: done, saving took 0.80 seconds 
+2022-04-12 04:16:22.666493: done 
+2022-04-12 04:16:22.691724: saving checkpoint... 
+2022-04-12 04:16:23.472085: done, saving took 0.81 seconds 
+2022-04-12 04:16:45.433445: This epoch took 513.239073 s
+ 
+2022-04-12 04:16:45.433738: 
+epoch:  88 
+2022-04-12 04:24:28.149167: train loss : 0.1043 
+2022-04-12 04:24:54.479588: validation loss: 0.1004 
+2022-04-12 04:24:54.480131: Average global foreground Dice: [0.6462, 0.8859] 
+2022-04-12 04:24:54.480213: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 04:24:54.885033: lr: 6.7e-05 
+2022-04-12 04:24:54.885221: saving scheduled checkpoint file... 
+2022-04-12 04:24:54.934930: saving checkpoint... 
+2022-04-12 04:24:55.815455: done, saving took 0.93 seconds 
+2022-04-12 04:24:55.823053: done 
+2022-04-12 04:24:55.850444: saving checkpoint... 
+2022-04-12 04:24:56.753325: done, saving took 0.93 seconds 
+2022-04-12 04:25:16.436562: This epoch took 511.002717 s
+ 
+2022-04-12 04:25:16.436848: 
+epoch:  89 
+2022-04-12 04:32:59.019767: train loss : 0.1034 
+2022-04-12 04:33:25.358483: validation loss: 0.1032 
+2022-04-12 04:33:25.359166: Average global foreground Dice: [0.6397, 0.8846] 
+2022-04-12 04:33:25.359251: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 04:33:25.760066: lr: 6.3e-05 
+2022-04-12 04:33:25.760276: saving scheduled checkpoint file... 
+2022-04-12 04:33:25.809910: saving checkpoint... 
+2022-04-12 04:33:26.703683: done, saving took 0.94 seconds 
+2022-04-12 04:33:26.712762: done 
+2022-04-12 04:33:26.745128: saving checkpoint... 
+2022-04-12 04:33:27.636033: done, saving took 0.92 seconds 
+2022-04-12 04:33:50.735580: This epoch took 514.298618 s
+ 
+2022-04-12 04:33:50.735859: 
+epoch:  90 
+2022-04-12 04:41:33.113122: train loss : 0.1008 
+2022-04-12 04:41:59.475420: validation loss: 0.0984 
+2022-04-12 04:41:59.475977: Average global foreground Dice: [0.6269, 0.884] 
+2022-04-12 04:41:59.476137: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 04:41:59.888294: lr: 6e-05 
+2022-04-12 04:41:59.888487: saving scheduled checkpoint file... 
+2022-04-12 04:41:59.939038: saving checkpoint... 
+2022-04-12 04:42:00.744348: done, saving took 0.86 seconds 
+2022-04-12 04:42:00.751153: done 
+2022-04-12 04:42:00.776982: saving checkpoint... 
+2022-04-12 04:42:01.575939: done, saving took 0.82 seconds 
+2022-04-12 04:42:18.097639: This epoch took 507.361677 s
+ 
+2022-04-12 04:42:18.097926: 
+epoch:  91 
+2022-04-12 04:50:01.081358: train loss : 0.0977 
+2022-04-12 04:50:27.449257: validation loss: 0.1181 
+2022-04-12 04:50:27.449831: Average global foreground Dice: [0.6423, 0.8551] 
+2022-04-12 04:50:27.449915: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 04:50:27.860837: lr: 5.7e-05 
+2022-04-12 04:50:27.861039: saving scheduled checkpoint file... 
+2022-04-12 04:50:27.910363: saving checkpoint... 
+2022-04-12 04:50:28.805149: done, saving took 0.94 seconds 
+2022-04-12 04:50:28.813156: done 
+2022-04-12 04:51:00.700607: This epoch took 522.602580 s
+ 
+2022-04-12 04:51:00.700882: 
+epoch:  92 
+2022-04-12 04:58:43.172932: train loss : 0.0993 
+2022-04-12 04:59:09.516165: validation loss: 0.1132 
+2022-04-12 04:59:09.516814: Average global foreground Dice: [0.6483, 0.8695] 
+2022-04-12 04:59:09.516901: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 04:59:09.920715: lr: 5.4e-05 
+2022-04-12 04:59:09.920972: saving scheduled checkpoint file... 
+2022-04-12 04:59:09.970634: saving checkpoint... 
+2022-04-12 04:59:10.865305: done, saving took 0.94 seconds 
+2022-04-12 04:59:10.873847: done 
+2022-04-12 04:59:10.909582: saving checkpoint... 
+2022-04-12 04:59:11.781749: done, saving took 0.91 seconds 
+2022-04-12 04:59:33.127198: This epoch took 512.426207 s
+ 
+2022-04-12 04:59:33.127510: 
+epoch:  93 
+2022-04-12 05:07:15.831490: train loss : 0.0940 
+2022-04-12 05:07:42.200351: validation loss: 0.1109 
+2022-04-12 05:07:42.200903: Average global foreground Dice: [0.6617, 0.8547] 
+2022-04-12 05:07:42.200985: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 05:07:42.599155: lr: 5.1e-05 
+2022-04-12 05:07:42.599353: saving scheduled checkpoint file... 
+2022-04-12 05:07:42.648796: saving checkpoint... 
+2022-04-12 05:07:43.536121: done, saving took 0.94 seconds 
+2022-04-12 05:07:43.544144: done 
+2022-04-12 05:07:43.572967: saving checkpoint... 
+2022-04-12 05:07:44.447019: done, saving took 0.90 seconds 
+2022-04-12 05:08:10.215270: This epoch took 517.087635 s
+ 
+2022-04-12 05:08:10.215595: 
+epoch:  94 
+2022-04-12 05:15:52.855393: train loss : 0.1015 
+2022-04-12 05:16:19.268202: validation loss: 0.1099 
+2022-04-12 05:16:19.268808: Average global foreground Dice: [0.6554, 0.8489] 
+2022-04-12 05:16:19.268892: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 05:16:19.672113: lr: 4.8e-05 
+2022-04-12 05:16:19.672306: saving scheduled checkpoint file... 
+2022-04-12 05:16:19.721741: saving checkpoint... 
+2022-04-12 05:16:20.529330: done, saving took 0.86 seconds 
+2022-04-12 05:16:20.558667: done 
+2022-04-12 05:16:58.649115: This epoch took 528.433414 s
+ 
+2022-04-12 05:16:58.649410: 
+epoch:  95 
+2022-04-12 05:24:40.946638: train loss : 0.0991 
+2022-04-12 05:25:07.301243: validation loss: 0.0969 
+2022-04-12 05:25:07.301813: Average global foreground Dice: [0.6551, 0.8487] 
+2022-04-12 05:25:07.301899: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 05:25:07.703476: lr: 4.5e-05 
+2022-04-12 05:25:07.703681: saving scheduled checkpoint file... 
+2022-04-12 05:25:07.753533: saving checkpoint... 
+2022-04-12 05:25:08.661253: done, saving took 0.96 seconds 
+2022-04-12 05:25:08.668943: done 
+2022-04-12 05:25:30.086483: This epoch took 511.436973 s
+ 
+2022-04-12 05:25:30.086766: 
+epoch:  96 
+2022-04-12 05:33:12.843010: train loss : 0.0983 
+2022-04-12 05:33:39.214464: validation loss: 0.1159 
+2022-04-12 05:33:39.215133: Average global foreground Dice: [0.6455, 0.8688] 
+2022-04-12 05:33:39.215220: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 05:33:39.631214: lr: 4.2e-05 
+2022-04-12 05:33:39.631405: saving scheduled checkpoint file... 
+2022-04-12 05:33:39.681313: saving checkpoint... 
+2022-04-12 05:33:40.582208: done, saving took 0.95 seconds 
+2022-04-12 05:33:40.590673: done 
+2022-04-12 05:34:24.072500: This epoch took 533.985631 s
+ 
+2022-04-12 05:34:24.072798: 
+epoch:  97 
+2022-04-12 05:42:06.595031: train loss : 0.0979 
+2022-04-12 05:42:32.949392: validation loss: 0.1080 
+2022-04-12 05:42:32.950503: Average global foreground Dice: [0.6516, 0.8672] 
+2022-04-12 05:42:32.950598: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 05:42:33.356525: lr: 3.9e-05 
+2022-04-12 05:42:33.356733: saving scheduled checkpoint file... 
+2022-04-12 05:42:33.406365: saving checkpoint... 
+2022-04-12 05:42:34.291275: done, saving took 0.93 seconds 
+2022-04-12 05:42:34.298696: done 
+2022-04-12 05:42:34.324865: saving checkpoint... 
+2022-04-12 05:42:35.196396: done, saving took 0.90 seconds 
+2022-04-12 05:43:01.950181: This epoch took 517.877281 s
+ 
+2022-04-12 05:43:01.950459: 
+epoch:  98 
+2022-04-12 05:50:44.144110: train loss : 0.0967 
+2022-04-12 05:51:10.501491: validation loss: 0.1040 
+2022-04-12 05:51:10.502162: Average global foreground Dice: [0.6688, 0.8406] 
+2022-04-12 05:51:10.502246: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 05:51:10.904269: lr: 3.6e-05 
+2022-04-12 05:51:10.904446: saving scheduled checkpoint file... 
+2022-04-12 05:51:10.953789: saving checkpoint... 
+2022-04-12 05:51:11.864342: done, saving took 0.96 seconds 
+2022-04-12 05:51:11.872040: done 
+2022-04-12 05:51:36.761628: This epoch took 514.811064 s
+ 
+2022-04-12 05:51:36.761946: 
+epoch:  99 
+2022-04-12 05:59:19.299343: train loss : 0.0981 
+2022-04-12 05:59:45.656408: validation loss: 0.1176 
+2022-04-12 05:59:45.656996: Average global foreground Dice: [0.6576, 0.8719] 
+2022-04-12 05:59:45.657082: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 05:59:46.057636: lr: 3.4e-05 
+2022-04-12 05:59:46.057829: saving scheduled checkpoint file... 
+2022-04-12 05:59:46.107279: saving checkpoint... 
+2022-04-12 05:59:46.958303: done, saving took 0.90 seconds 
+2022-04-12 05:59:46.966263: done 
+2022-04-12 05:59:46.993814: saving checkpoint... 
+2022-04-12 05:59:47.861776: done, saving took 0.90 seconds 
+2022-04-12 06:00:07.657879: This epoch took 510.895798 s
+ 
+2022-04-12 06:00:07.658240: 
+epoch:  100 
+2022-04-12 06:07:50.112843: train loss : 0.1014 
+2022-04-12 06:08:16.516289: validation loss: 0.1085 
+2022-04-12 06:08:16.516893: Average global foreground Dice: [0.6732, 0.882] 
+2022-04-12 06:08:16.516977: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 06:08:16.930567: lr: 3.1e-05 
+2022-04-12 06:08:16.930767: saving scheduled checkpoint file... 
+2022-04-12 06:08:16.980258: saving checkpoint... 
+2022-04-12 06:08:17.801977: done, saving took 0.87 seconds 
+2022-04-12 06:08:17.809429: done 
+2022-04-12 06:08:17.836146: saving checkpoint... 
+2022-04-12 06:08:18.598226: done, saving took 0.79 seconds 
+2022-04-12 06:08:39.009843: This epoch took 511.351490 s
+ 
+2022-04-12 06:08:39.010150: 
+epoch:  101 
+2022-04-12 06:16:21.644188: train loss : 0.0946 
+2022-04-12 06:16:48.017226: validation loss: 0.1126 
+2022-04-12 06:16:48.017861: Average global foreground Dice: [0.6545, 0.8668] 
+2022-04-12 06:16:48.017947: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 06:16:48.422889: lr: 2.9e-05 
+2022-04-12 06:16:48.423091: saving scheduled checkpoint file... 
+2022-04-12 06:16:48.472787: saving checkpoint... 
+2022-04-12 06:16:49.352198: done, saving took 0.93 seconds 
+2022-04-12 06:16:49.360469: done 
+2022-04-12 06:16:49.389912: saving checkpoint... 
+2022-04-12 06:16:50.275704: done, saving took 0.92 seconds 
+2022-04-12 06:17:13.655284: This epoch took 514.645032 s
+ 
+2022-04-12 06:17:13.655623: 
+epoch:  102 
+2022-04-12 06:24:56.514107: train loss : 0.0948 
+2022-04-12 06:25:22.929111: validation loss: 0.1128 
+2022-04-12 06:25:22.929702: Average global foreground Dice: [0.6534, 0.8651] 
+2022-04-12 06:25:22.929787: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 06:25:23.345099: lr: 2.6e-05 
+2022-04-12 06:25:23.345296: saving scheduled checkpoint file... 
+2022-04-12 06:25:23.394982: saving checkpoint... 
+2022-04-12 06:25:24.291731: done, saving took 0.95 seconds 
+2022-04-12 06:25:24.299320: done 
+2022-04-12 06:25:24.325481: saving checkpoint... 
+2022-04-12 06:25:25.198053: done, saving took 0.90 seconds 
+2022-04-12 06:26:04.287975: This epoch took 530.632249 s
+ 
+2022-04-12 06:26:04.288282: 
+epoch:  103 
+2022-04-12 06:33:47.260315: train loss : 0.0910 
+2022-04-12 06:34:13.618779: validation loss: 0.1122 
+2022-04-12 06:34:13.619409: Average global foreground Dice: [0.6486, 0.8714] 
+2022-04-12 06:34:13.619493: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 06:34:14.021538: lr: 2.4e-05 
+2022-04-12 06:34:14.021744: saving scheduled checkpoint file... 
+2022-04-12 06:34:14.071274: saving checkpoint... 
+2022-04-12 06:34:14.890485: done, saving took 0.87 seconds 
+2022-04-12 06:34:14.897752: done 
+2022-04-12 06:34:14.923895: saving checkpoint... 
+2022-04-12 06:34:15.695315: done, saving took 0.80 seconds 
+2022-04-12 06:34:35.904371: This epoch took 511.615988 s
+ 
+2022-04-12 06:34:35.904716: 
+epoch:  104 
+2022-04-12 06:42:18.150730: train loss : 0.0937 
+2022-04-12 06:42:44.539033: validation loss: 0.1077 
+2022-04-12 06:42:44.539643: Average global foreground Dice: [0.6593, 0.8761] 
+2022-04-12 06:42:44.539728: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 06:42:44.946461: lr: 2.2e-05 
+2022-04-12 06:42:44.946668: saving scheduled checkpoint file... 
+2022-04-12 06:42:44.996276: saving checkpoint... 
+2022-04-12 06:42:45.880019: done, saving took 0.93 seconds 
+2022-04-12 06:42:45.887694: done 
+2022-04-12 06:42:45.926809: saving checkpoint... 
+2022-04-12 06:42:46.787819: done, saving took 0.90 seconds 
+2022-04-12 06:43:03.784698: This epoch took 507.879827 s
+ 
+2022-04-12 06:43:03.784977: 
+epoch:  105 
+2022-04-12 06:50:46.285807: train loss : 0.0888 
+2022-04-12 06:51:12.650251: validation loss: 0.1235 
+2022-04-12 06:51:12.650828: Average global foreground Dice: [0.647, 0.8546] 
+2022-04-12 06:51:12.650913: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 06:51:13.055029: lr: 2e-05 
+2022-04-12 06:51:13.055219: saving scheduled checkpoint file... 
+2022-04-12 06:51:13.105167: saving checkpoint... 
+2022-04-12 06:51:14.011938: done, saving took 0.96 seconds 
+2022-04-12 06:51:14.020160: done 
+2022-04-12 06:51:30.358945: This epoch took 506.573867 s
+ 
+2022-04-12 06:51:30.359259: 
+epoch:  106 
+2022-04-12 06:59:12.830997: train loss : 0.0939 
+2022-04-12 06:59:39.170909: validation loss: 0.1194 
+2022-04-12 06:59:39.171612: Average global foreground Dice: [0.6563, 0.8687] 
+2022-04-12 06:59:39.171698: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 06:59:39.576355: lr: 1.8e-05 
+2022-04-12 06:59:39.576545: saving scheduled checkpoint file... 
+2022-04-12 06:59:39.628755: saving checkpoint... 
+2022-04-12 06:59:40.511643: done, saving took 0.93 seconds 
+2022-04-12 06:59:40.519433: done 
+2022-04-12 07:00:05.455869: This epoch took 515.096512 s
+ 
+2022-04-12 07:00:05.456148: 
+epoch:  107 
+2022-04-12 07:07:48.161068: train loss : 0.0921 
+2022-04-12 07:08:14.469932: validation loss: 0.1120 
+2022-04-12 07:08:14.470494: Average global foreground Dice: [0.6492, 0.8366] 
+2022-04-12 07:08:14.470588: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 07:08:14.874798: lr: 1.6e-05 
+2022-04-12 07:08:14.874981: saving scheduled checkpoint file... 
+2022-04-12 07:08:14.924342: saving checkpoint... 
+2022-04-12 07:08:15.680308: done, saving took 0.81 seconds 
+2022-04-12 07:08:15.687589: done 
+2022-04-12 07:08:41.636631: This epoch took 516.180385 s
+ 
+2022-04-12 07:08:41.636923: 
+epoch:  108 
+2022-04-12 07:16:24.231153: train loss : 0.0926 
+2022-04-12 07:16:50.641657: validation loss: 0.1105 
+2022-04-12 07:16:50.642201: Average global foreground Dice: [0.6579, 0.8562] 
+2022-04-12 07:16:50.642281: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 07:16:51.045394: lr: 1.4e-05 
+2022-04-12 07:16:51.045588: saving scheduled checkpoint file... 
+2022-04-12 07:16:51.094967: saving checkpoint... 
+2022-04-12 07:16:51.972413: done, saving took 0.93 seconds 
+2022-04-12 07:16:51.979677: done 
+2022-04-12 07:17:13.522203: This epoch took 511.885176 s
+ 
+2022-04-12 07:17:13.522489: 
+epoch:  109 
+2022-04-12 07:24:55.869624: train loss : 0.0985 
+2022-04-12 07:25:22.211050: validation loss: 0.1209 
+2022-04-12 07:25:22.211686: Average global foreground Dice: [0.6453, 0.856] 
+2022-04-12 07:25:22.211768: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 07:25:22.622737: lr: 1.2e-05 
+2022-04-12 07:25:22.622922: saving scheduled checkpoint file... 
+2022-04-12 07:25:22.672471: saving checkpoint... 
+2022-04-12 07:25:23.581385: done, saving took 0.96 seconds 
+2022-04-12 07:25:23.588823: done 
+2022-04-12 07:25:48.872918: This epoch took 515.350323 s
+ 
+2022-04-12 07:25:48.873195: 
+epoch:  110 
+2022-04-12 07:33:32.551534: train loss : 0.0973 
+2022-04-12 07:33:58.899036: validation loss: 0.1142 
+2022-04-12 07:33:58.899562: Average global foreground Dice: [0.6618, 0.8648] 
+2022-04-12 07:33:58.899647: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 07:33:59.307953: lr: 1.1e-05 
+2022-04-12 07:33:59.308133: saving scheduled checkpoint file... 
+2022-04-12 07:33:59.357507: saving checkpoint... 
+2022-04-12 07:34:00.162388: done, saving took 0.85 seconds 
+2022-04-12 07:34:00.169318: done 
+2022-04-12 07:34:18.244838: This epoch took 509.371542 s
+ 
+2022-04-12 07:34:18.245121: 
+epoch:  111 
+2022-04-12 07:42:00.635880: train loss : 0.0909 
+2022-04-12 07:42:27.000191: validation loss: 0.1107 
+2022-04-12 07:42:27.000841: Average global foreground Dice: [0.6604, 0.8594] 
+2022-04-12 07:42:27.000930: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 07:42:27.404088: lr: 9e-06 
+2022-04-12 07:42:27.404283: saving scheduled checkpoint file... 
+2022-04-12 07:42:27.453604: saving checkpoint... 
+2022-04-12 07:42:28.342470: done, saving took 0.94 seconds 
+2022-04-12 07:42:28.350491: done 
+2022-04-12 07:42:50.313202: This epoch took 512.067981 s
+ 
+2022-04-12 07:42:50.313535: 
+epoch:  112 
+2022-04-12 07:50:32.778379: train loss : 0.1013 
+2022-04-12 07:50:59.161238: validation loss: 0.1168 
+2022-04-12 07:50:59.161930: Average global foreground Dice: [0.6706, 0.8804] 
+2022-04-12 07:50:59.162031: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 07:50:59.578756: lr: 8e-06 
+2022-04-12 07:50:59.578949: saving scheduled checkpoint file... 
+2022-04-12 07:50:59.628338: saving checkpoint... 
+2022-04-12 07:51:00.521036: done, saving took 0.94 seconds 
+2022-04-12 07:51:00.529813: done 
+2022-04-12 07:51:20.023951: This epoch took 509.710284 s
+ 
+2022-04-12 07:51:20.024251: 
+epoch:  113 
+2022-04-12 07:59:02.259558: train loss : 0.0952 
+2022-04-12 07:59:28.574529: validation loss: 0.1126 
+2022-04-12 07:59:28.575164: Average global foreground Dice: [0.6641, 0.8765] 
+2022-04-12 07:59:28.575247: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 07:59:28.994439: lr: 7e-06 
+2022-04-12 07:59:28.994648: saving scheduled checkpoint file... 
+2022-04-12 07:59:29.044352: saving checkpoint... 
+2022-04-12 07:59:29.917429: done, saving took 0.92 seconds 
+2022-04-12 07:59:29.924901: done 
+2022-04-12 07:59:29.951155: saving checkpoint... 
+2022-04-12 07:59:30.825511: done, saving took 0.90 seconds 
+2022-04-12 08:00:04.223475: This epoch took 524.199126 s
+ 
+2022-04-12 08:00:04.223759: 
+epoch:  114 
+2022-04-12 08:07:47.044150: train loss : 0.0915 
+2022-04-12 08:08:13.393082: validation loss: 0.0974 
+2022-04-12 08:08:13.393772: Average global foreground Dice: [0.6534, 0.8701] 
+2022-04-12 08:08:13.393862: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 08:08:13.804751: lr: 6e-06 
+2022-04-12 08:08:13.804959: saving scheduled checkpoint file... 
+2022-04-12 08:08:13.855244: saving checkpoint... 
+2022-04-12 08:08:14.610067: done, saving took 0.81 seconds 
+2022-04-12 08:08:14.618208: done 
+2022-04-12 08:08:14.645764: saving checkpoint... 
+2022-04-12 08:08:15.449506: done, saving took 0.83 seconds 
+2022-04-12 08:08:31.929602: This epoch took 507.705730 s
+ 
+2022-04-12 08:08:31.929906: 
+epoch:  115 
+2022-04-12 08:16:15.132812: train loss : 0.0947 
+2022-04-12 08:16:41.487801: validation loss: 0.1080 
+2022-04-12 08:16:41.488466: Average global foreground Dice: [0.6604, 0.8744] 
+2022-04-12 08:16:41.488559: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 08:16:41.899711: lr: 5e-06 
+2022-04-12 08:16:41.899907: saving scheduled checkpoint file... 
+2022-04-12 08:16:41.949359: saving checkpoint... 
+2022-04-12 08:16:42.869321: done, saving took 0.97 seconds 
+2022-04-12 08:16:42.877380: done 
+2022-04-12 08:16:42.906241: saving checkpoint... 
+2022-04-12 08:16:43.814054: done, saving took 0.94 seconds 
+2022-04-12 08:17:08.953405: This epoch took 517.023400 s
+ 
+2022-04-12 08:17:08.953717: 
+epoch:  116 
+2022-04-12 08:24:52.541597: train loss : 0.0937 
+2022-04-12 08:25:18.884503: validation loss: 0.1244 
+2022-04-12 08:25:18.885070: Average global foreground Dice: [0.6509, 0.8797] 
+2022-04-12 08:25:18.885158: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 08:25:19.299258: lr: 4e-06 
+2022-04-12 08:25:19.299438: saving scheduled checkpoint file... 
+2022-04-12 08:25:19.348815: saving checkpoint... 
+2022-04-12 08:25:20.245363: done, saving took 0.95 seconds 
+2022-04-12 08:25:20.253742: done 
+2022-04-12 08:25:20.282433: saving checkpoint... 
+2022-04-12 08:25:21.142138: done, saving took 0.89 seconds 
+2022-04-12 08:25:45.637394: This epoch took 516.683575 s
+ 
+2022-04-12 08:25:45.637714: 
+epoch:  117 
+2022-04-12 08:33:28.370584: train loss : 0.0921 
+2022-04-12 08:33:54.730809: validation loss: 0.1215 
+2022-04-12 08:33:54.731369: Average global foreground Dice: [0.6466, 0.8382] 
+2022-04-12 08:33:54.731454: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 08:33:55.139373: lr: 3e-06 
+2022-04-12 08:33:55.139580: saving scheduled checkpoint file... 
+2022-04-12 08:33:55.189398: saving checkpoint... 
+2022-04-12 08:33:56.087631: done, saving took 0.95 seconds 
+2022-04-12 08:33:56.095191: done 
+2022-04-12 08:34:26.783720: This epoch took 521.145902 s
+ 
+2022-04-12 08:34:26.784005: 
+epoch:  118 
+2022-04-12 08:42:09.272538: train loss : 0.0911 
+2022-04-12 08:42:35.604627: validation loss: 0.1132 
+2022-04-12 08:42:35.605283: Average global foreground Dice: [0.653, 0.8752] 
+2022-04-12 08:42:35.605367: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 08:42:36.014820: lr: 2e-06 
+2022-04-12 08:42:36.015008: saving scheduled checkpoint file... 
+2022-04-12 08:42:36.065012: saving checkpoint... 
+2022-04-12 08:42:36.967463: done, saving took 0.95 seconds 
+2022-04-12 08:42:36.974240: done 
+2022-04-12 08:43:25.452103: This epoch took 538.667975 s
+ 
+2022-04-12 08:43:25.452703: 
+epoch:  119 
+2022-04-12 08:51:08.222655: train loss : 0.0946 
+2022-04-12 08:51:34.558525: validation loss: 0.1305 
+2022-04-12 08:51:34.559131: Average global foreground Dice: [0.6626, 0.8591] 
+2022-04-12 08:51:34.559214: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 08:51:34.983504: lr: 1e-06 
+2022-04-12 08:51:34.983703: saving scheduled checkpoint file... 
+2022-04-12 08:51:35.033106: saving checkpoint... 
+2022-04-12 08:51:35.926189: done, saving took 0.94 seconds 
+2022-04-12 08:51:35.933503: done 
+2022-04-12 08:51:58.995100: This epoch took 513.542206 s
+ 
+2022-04-12 08:51:58.995444: 
+epoch:  120 
+2022-04-12 08:59:41.557760: train loss : 0.0925 
+2022-04-12 09:00:07.862391: validation loss: 0.0994 
+2022-04-12 09:00:07.863036: Average global foreground Dice: [0.6634, 0.885] 
+2022-04-12 09:00:07.863123: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 09:00:08.269758: lr: 1e-06 
+2022-04-12 09:00:08.269943: saving scheduled checkpoint file... 
+2022-04-12 09:00:08.319494: saving checkpoint... 
+2022-04-12 09:00:09.144474: done, saving took 0.87 seconds 
+2022-04-12 09:00:09.151906: done 
+2022-04-12 09:00:39.517333: This epoch took 520.521788 s
+ 
+2022-04-12 09:00:39.517647: 
+epoch:  121 
+2022-04-12 09:08:22.503497: train loss : 0.0988 
+2022-04-12 09:08:48.851639: validation loss: 0.1012 
+2022-04-12 09:08:48.852184: Average global foreground Dice: [0.664, 0.8654] 
+2022-04-12 09:08:48.852267: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 09:08:49.256476: lr: 1e-06 
+2022-04-12 09:08:49.256670: saving scheduled checkpoint file... 
+2022-04-12 09:08:49.305947: saving checkpoint... 
+2022-04-12 09:08:50.180798: done, saving took 0.92 seconds 
+2022-04-12 09:08:50.189090: done 
+2022-04-12 09:08:50.218187: saving checkpoint... 
+2022-04-12 09:08:51.111200: done, saving took 0.92 seconds 
+2022-04-12 09:09:13.634463: This epoch took 514.116712 s
+ 
+2022-04-12 09:09:13.634751: 
+epoch:  122 
+2022-04-12 09:16:56.053530: train loss : 0.0965 
+2022-04-12 09:17:22.378980: validation loss: 0.1227 
+2022-04-12 09:17:22.379484: Average global foreground Dice: [0.6516, 0.881] 
+2022-04-12 09:17:22.379575: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 09:17:22.788263: lr: 0.0 
+2022-04-12 09:17:22.788441: saving scheduled checkpoint file... 
+2022-04-12 09:17:22.837835: saving checkpoint... 
+2022-04-12 09:17:23.742558: done, saving took 0.95 seconds 
+2022-04-12 09:17:23.750092: done 
+2022-04-12 09:17:23.776498: saving checkpoint... 
+2022-04-12 09:17:24.676388: done, saving took 0.93 seconds 
+2022-04-12 09:17:53.596542: This epoch took 519.961684 s
+ 
+2022-04-12 09:17:53.596852: 
+epoch:  123 
+2022-04-12 09:25:36.507234: train loss : 0.0970 
+2022-04-12 09:26:02.863523: validation loss: 0.1171 
+2022-04-12 09:26:02.864114: Average global foreground Dice: [0.6551, 0.8827] 
+2022-04-12 09:26:02.864197: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 09:26:03.272641: lr: 0.0 
+2022-04-12 09:26:03.272827: saving scheduled checkpoint file... 
+2022-04-12 09:26:03.322485: saving checkpoint... 
+2022-04-12 09:26:04.107136: done, saving took 0.83 seconds 
+2022-04-12 09:26:04.128266: done 
+2022-04-12 09:26:04.154564: saving checkpoint... 
+2022-04-12 09:26:04.900265: done, saving took 0.77 seconds 
+2022-04-12 09:26:25.513794: This epoch took 511.916837 s
+ 
+2022-04-12 09:26:25.514117: 
+epoch:  124 
+2022-04-12 09:34:08.067902: train loss : 0.0985 
+2022-04-12 09:34:34.424941: validation loss: 0.1139 
+2022-04-12 09:34:34.425489: Average global foreground Dice: [0.647, 0.8681] 
+2022-04-12 09:34:34.425583: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-12 09:34:34.832934: lr: 0.0 
+2022-04-12 09:34:34.833128: saving scheduled checkpoint file... 
+2022-04-12 09:34:34.882842: saving checkpoint... 
+2022-04-12 09:34:35.771612: done, saving took 0.94 seconds 
+2022-04-12 09:34:35.779083: done 
+2022-04-12 09:35:15.461941: This epoch took 529.947685 s
+ 
+2022-04-12 09:35:15.520846: saving checkpoint... 
+2022-04-12 09:35:16.280211: done, saving took 0.82 seconds 

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/debug.json

@@ -0,0 +1,106 @@
+{
+    "all_tr_losses": "[0.3365588, 0.23576154, 0.20644811, 0.1925307, 0.17650223, 0.17113653, 0.15389562, 0.15411775, 0.14826968, 0.15403421, 0.1440485, 0.15402897, 0.14373074, 0.14662163, 0.14329961, 0.13333677, 0.14521024, 0.13089782, 0.13481817, 0.12698181, 0.13424017, 0.12435664, 0.12738469, 0.13200112, 0.1282572, 0.13086745, 0.12717266, 0.13209039, 0.11893695, 0.12243814, 0.12271249, 0.1244644, 0.120372586, 0.1278699, 0.1222408, 0.12434122, 0.121071704, 0.115636334, 0.12998971, 0.12240019, 0.117289364, 0.11434085, 0.1206092, 0.11047681, 0.11596647, 0.12526226, 0.116367556, 0.113776915, 0.11709769, 0.111778274, 0.11058851, 0.11627464, 0.11668352, 0.11725866, 0.114777334, 0.10902945, 0.11752333, 0.105974056, 0.11599809, 0.11460744, 0.11100236, 0.11232076, 0.11299875, 0.11548251, 0.105052, 0.10675668, 0.10548254, 0.11169742, 0.1080365, 0.11105766, 0.09920337, 0.103396885, 0.10165332, 0.10527045, 0.10346537, 0.108782746, 0.1044361, 0.09820995, 0.09989239, 0.09823276, 0.10304799, 0.100780904, 0.103387326, 0.09280554, 0.10075345, 0.09364079, 0.09474833, 0.097541355, 0.096883245, 0.09801272, 0.08907745, 0.094783954, 0.0900567]",
+    "all_val_eval_metrics": "[0.22362134851066784, 0.4681743519413855, 0.4763226412707299, 0.5314078850580991, 0.5855597511427186, 0.5573378363123547, 0.6144453233639567, 0.6361954419336993, 0.6678669186170278, 0.6636059055274977, 0.6609184350362479, 0.4689656073727865, 0.6380004375952435, 0.6937566438448917, 0.7016893401201112, 0.6953145003294778, 0.7078793778405844, 0.6964784489636958, 0.7039896611165055, 0.6837297054472242, 0.6976557239425736, 0.6623992889203882, 0.6938693226127461, 0.7319395860419855, 0.7048480263902688, 0.7034690476808928, 0.729154117172781, 0.7219168182442416, 0.7138226212201069, 0.7265563622597072, 0.7260280876057625, 0.7245606878544991, 0.7210457545958671, 0.7215736167545345, 0.7280911293911866, 0.7236171709093452, 0.7357761250722543, 0.7148970244206554, 0.7020775640464304, 0.7184920115418894, 0.7400005779958321, 0.7364482643579509, 0.7460976597333615, 0.7447956048099591, 0.7277171365864633, 0.7174136244776235, 0.7332727298428101, 0.7119007579894789, 0.728118948650897, 0.7250240898958595, 0.7264547569968802, 0.7479807567826615, 0.7271300484412927, 0.7364948036735464, 0.7507829211833852, 0.7431158993354015, 0.7276440741421553, 0.7441998454281455, 0.7348914376561888, 0.7441568384881951, 0.7417610191775222, 0.7325026434681247, 0.7512153099450354, 0.7530621122398669, 0.7566591009206811, 0.7543712305846757, 0.7504239165067992, 0.739429560895169, 0.7417044956448609, 0.7428402452453265, 0.7254917800449425, 0.7510673725543584, 0.7486325422270188, 0.7520073425038944, 0.7528195565482716, 0.7385722561634516, 0.7417615219162972, 0.7589276364374139, 0.7501715194456285, 0.7511627793027655, 0.7336959785489074, 0.7346514273128679, 0.738454071100811, 0.7210844771792085, 0.7519652776191077, 0.7551270491149819, 0.7525788386506502, 0.7324255515490765, 0.7332737453884888, 0.7389992681438778, 0.719622482794996, 0.7474999850059756, 0.7512417061933725]",
+    "all_val_losses": "[0.27353683, 0.21563168, 0.22545616, 0.17471375, 0.16070622, 0.16916835, 0.14368363, 0.14897588, 0.14481923, 0.14669828, 0.16085072, 0.17799395, 0.13749734, 0.1578951, 0.1339963, 0.13933295, 0.13721834, 0.14562169, 0.13580345, 0.14007284, 0.12492545, 0.15388352, 0.12720071, 0.13041332, 0.1352024, 0.12542722, 0.12870632, 0.14012887, 0.11813672, 0.11324752, 0.123189956, 0.12444647, 0.13844359, 0.12112669, 0.119424894, 0.13957475, 0.11372627, 0.13046922, 0.13683175, 0.13842992, 0.122694634, 0.12575035, 0.10276161, 0.12905096, 0.13321823, 0.12823167, 0.1158135, 0.14339434, 0.122966096, 0.121158324, 0.12623544, 0.11132485, 0.10824719, 0.12059457, 0.12660325, 0.13544, 0.13883731, 0.12793529, 0.13314013, 0.116851464, 0.11731087, 0.14206949, 0.13704526, 0.115740165, 0.10180616, 0.11520993, 0.1501946, 0.1139482, 0.12092621, 0.1256443, 0.11076236, 0.10439913, 0.107668, 0.11303625, 0.12650496, 0.11198064, 0.13196927, 0.10110061, 0.113442555, 0.11976511, 0.11249426, 0.109426945, 0.12533823, 0.11059702, 0.10330965, 0.11261508, 0.11954655, 0.11479926, 0.109553464, 0.12314592, 0.11573183, 0.1017501, 0.100643724]",
+    "all_val_losses_tr_mode": "[]",
+    "also_val_in_tr_mode": "False",
+    "amp_grad_scaler": "<torch.cuda.amp.grad_scaler.GradScaler object at 0x7f46a8e0e160>",
+    "args": "Namespace(batch_size=2, config='./configs/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32.yaml', continue_training=False, crop_size=[64, 192, 192], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='2', fp32=False, hdfs_base='GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=125, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task008_HepaticVessel', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, vallatest=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)",
+    "base_num_features": "32",
+    "basic_generator_patch_size": "[178 308 233]",
+    "batch_dice": "True",
+    "batch_size": "2",
+    "best_MA_tr_loss_for_patience": "None",
+    "best_epoch_based_on_MA_tr_loss": "None",
+    "best_val_eval_criterion_MA": "None",
+    "classes": "[1, 2]",
+    "conv_per_stage": "2",
+    "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-0.5235987755982988, 0.5235987755982988), 'rotation_y': (-0.5235987755982988, 0.5235987755982988), 'rotation_z': (-0.5235987755982988, 0.5235987755982988), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([ 64, 192, 192])}",
+    "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel",
+    "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]",
+    "deterministic": "False",
+    "disable_ds": "True",
+    "distribute_batch_size": "False",
+    "dl_tr": "<dataset_loading.DataLoader3D object at 0x7f468efdfbe0>",
+    "dl_val": "<dataset_loading.DataLoader3D object at 0x7f468efdf780>",
+    "do_dummy_2D_aug": "False",
+    "ds_loss_weights": "[1. 0. 0. 0. 0.]",
+    "epoch": "93",
+    "experiment_name": "nnUNetTrainerV2_DDP",
+    "find_zero_weight_decay": "False",
+    "fold": "2",
+    "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetData_plans_v2.1_stage1",
+    "fp16": "True",
+    "global_batch_size": "16",
+    "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/gt_segmentations",
+    "inference_pad_border_mode": "constant",
+    "inference_pad_kwargs": "{'constant_values': 0}",
+    "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl', 2, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', '/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel', True, 1, True, False, False, True)",
+    "initial_lr": "0.0003",
+    "input_size": "[64, 192, 192]",
+    "is_spatial_aug_only": "False",
+    "is_ssl": "False",
+    "layer_decay": "1.0",
+    "local_rank": "0",
+    "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/training_log_2022_5_27_13_08_39.txt",
+    "loss": "None",
+    "lr_scheduler": "<optimizers.lr_scheduler.LinearWarmupCosineAnnealingLR object at 0x7f464402a860>",
+    "lr_scheduler_eps": "0.001",
+    "lr_scheduler_name": "warmup_cosine",
+    "lr_scheduler_patience": "30",
+    "lr_threshold": "1e-06",
+    "max_num_epochs": "125",
+    "min_lr": "0",
+    "min_region_size_per_class": "None",
+    "min_size_per_class": "None",
+    "model": "Generic_TransUNet_max_ppbp",
+    "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}",
+    "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]",
+    "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]]",
+    "net_pool_per_axis": "[4, 5, 5]",
+    "normalization_schemes": "OrderedDict([(0, 'CT')])",
+    "num_batches_per_epoch": "250",
+    "num_classes": "3",
+    "num_input_channels": "1",
+    "num_val_batches_per_epoch": "50",
+    "online_eval_fn": "[]",
+    "online_eval_foreground_dc": "[]",
+    "online_eval_fp": "[]",
+    "online_eval_tp": "[]",
+    "only_keep_largest_connected_component": "None",
+    "optim_name": "adamw",
+    "optimizer": "AdamW (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    eps: 1e-08\n    initial_lr: 0.0003\n    lr: 5.375636079717583e-05\n    weight_decay: 3e-05\n)",
+    "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2",
+    "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32",
+    "oversample_foreground_percent": "0.0",
+    "pad_all_sides": "None",
+    "patch_size": "[ 64 192 192]",
+    "patience": "50",
+    "pin_memory": "True",
+    "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl",
+    "reclip": "None",
+    "regions_class_order": "None",
+    "resume": "auto",
+    "save_best_checkpoint": "True",
+    "save_every": "1",
+    "save_final_checkpoint": "True",
+    "save_intermediate_checkpoints": "True",
+    "save_latest_only": "True",
+    "stage": "1",
+    "threeD": "True",
+    "tr_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7f468f206b00>",
+    "train_loss_MA": "None",
+    "train_loss_MA_alpha": "0.93",
+    "train_loss_MA_eps": "0.0005",
+    "transpose_backward": "[0, 1, 2]",
+    "transpose_forward": "[0, 1, 2]",
+    "unpack_data": "True",
+    "use_mask_for_norm": "OrderedDict([(0, False)])",
+    "use_progress_bar": "False",
+    "val_eval_criterion_MA": "None",
+    "val_eval_criterion_alpha": "0.9",
+    "val_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7f468ed01358>",
+    "warmup_epochs": "10",
+    "was_initialized": "True",
+    "weight_decay": "3e-05"
+}

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_best.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fc7152cc5e27f4e249045e3e98919a5c20655607eb191d22fca620da37b48dd
+size 496524090

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_best.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a492b355d6b010096d22ee9fd14ac1404f535476bae2db246e26e0d73067aa2f
+size 188932

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_final_checkpoint.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e02586e63084544c716d993cd695a3a03341818bd602b274da8d84e2b9fb31e
+size 496529466

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_final_checkpoint.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a492b355d6b010096d22ee9fd14ac1404f535476bae2db246e26e0d73067aa2f
+size 188932

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_latest.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f3d23b0960aaccc552a5d98571138a4a0bcb004acbd1046712386270088d21e
+size 496529466

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_latest.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a492b355d6b010096d22ee9fd14ac1404f535476bae2db246e26e0d73067aa2f
+size 188932

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/training_log_2022_5_27_13_08_39.txt

@@ -0,0 +1,433 @@
+Starting... 
+2022-05-27 13:08:39.986044: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/splits_final.pkl 
+2022-05-27 13:08:39.986553: The split file contains 5 splits. 
+2022-05-27 13:08:39.986617: Desired fold for training: 2 
+2022-05-27 13:08:39.986657: This split has 242 training and 61 validation cases. 
+2022-05-27 13:10:18.506199: TRAINING KEYS:
+ odict_keys(['hepaticvessel_001', 'hepaticvessel_005', 'hepaticvessel_008', 'hepaticvessel_010', 'hepaticvessel_011', 'hepaticvessel_013', 'hepaticvessel_016', 'hepaticvessel_018', 'hepaticvessel_019', 'hepaticvessel_020', 'hepaticvessel_023', 'hepaticvessel_025', 'hepaticvessel_026', 'hepaticvessel_027', 'hepaticvessel_028', 'hepaticvessel_029', 'hepaticvessel_030', 'hepaticvessel_031', 'hepaticvessel_032', 'hepaticvessel_033', 'hepaticvessel_044', 'hepaticvessel_049', 'hepaticvessel_050', 'hepaticvessel_052', 'hepaticvessel_053', 'hepaticvessel_057', 'hepaticvessel_058', 'hepaticvessel_059', 'hepaticvessel_061', 'hepaticvessel_062', 'hepaticvessel_065', 'hepaticvessel_066', 'hepaticvessel_067', 'hepaticvessel_070', 'hepaticvessel_071', 'hepaticvessel_072', 'hepaticvessel_075', 'hepaticvessel_077', 'hepaticvessel_079', 'hepaticvessel_080', 'hepaticvessel_082', 'hepaticvessel_083', 'hepaticvessel_084', 'hepaticvessel_085', 'hepaticvessel_087', 'hepaticvessel_088', 'hepaticvessel_089', 'hepaticvessel_090', 'hepaticvessel_092', 'hepaticvessel_093', 'hepaticvessel_094', 'hepaticvessel_096', 'hepaticvessel_098', 'hepaticvessel_100', 'hepaticvessel_102', 'hepaticvessel_103', 'hepaticvessel_104', 'hepaticvessel_108', 'hepaticvessel_110', 'hepaticvessel_111', 'hepaticvessel_112', 'hepaticvessel_113', 'hepaticvessel_115', 'hepaticvessel_119', 'hepaticvessel_121', 'hepaticvessel_123', 'hepaticvessel_124', 'hepaticvessel_127', 'hepaticvessel_129', 'hepaticvessel_131', 'hepaticvessel_132', 'hepaticvessel_133', 'hepaticvessel_136', 'hepaticvessel_142', 'hepaticvessel_145', 'hepaticvessel_146', 'hepaticvessel_147', 'hepaticvessel_148', 'hepaticvessel_150', 'hepaticvessel_154', 'hepaticvessel_157', 'hepaticvessel_158', 'hepaticvessel_159', 'hepaticvessel_161', 'hepaticvessel_163', 'hepaticvessel_164', 'hepaticvessel_165', 'hepaticvessel_166', 'hepaticvessel_167', 'hepaticvessel_171', 'hepaticvessel_173', 'hepaticvessel_174', 'hepaticvessel_175', 'hepaticvessel_177', 'hepaticvessel_179', 'hepaticvessel_181', 'hepaticvessel_183', 'hepaticvessel_185', 'hepaticvessel_186', 'hepaticvessel_189', 'hepaticvessel_192', 'hepaticvessel_194', 'hepaticvessel_195', 'hepaticvessel_196', 'hepaticvessel_198', 'hepaticvessel_199', 'hepaticvessel_200', 'hepaticvessel_206', 'hepaticvessel_207', 'hepaticvessel_208', 'hepaticvessel_210', 'hepaticvessel_211', 'hepaticvessel_213', 'hepaticvessel_215', 'hepaticvessel_217', 'hepaticvessel_218', 'hepaticvessel_219', 'hepaticvessel_221', 'hepaticvessel_222', 'hepaticvessel_223', 'hepaticvessel_224', 'hepaticvessel_225', 'hepaticvessel_227', 'hepaticvessel_230', 'hepaticvessel_231', 'hepaticvessel_234', 'hepaticvessel_235', 'hepaticvessel_237', 'hepaticvessel_240', 'hepaticvessel_242', 'hepaticvessel_244', 'hepaticvessel_245', 'hepaticvessel_246', 'hepaticvessel_248', 'hepaticvessel_255', 'hepaticvessel_256', 'hepaticvessel_258', 'hepaticvessel_259', 'hepaticvessel_262', 'hepaticvessel_266', 'hepaticvessel_269', 'hepaticvessel_270', 'hepaticvessel_271', 'hepaticvessel_272', 'hepaticvessel_274', 'hepaticvessel_275', 'hepaticvessel_278', 'hepaticvessel_279', 'hepaticvessel_281', 'hepaticvessel_282', 'hepaticvessel_284', 'hepaticvessel_285', 'hepaticvessel_287', 'hepaticvessel_290', 'hepaticvessel_291', 'hepaticvessel_293', 'hepaticvessel_294', 'hepaticvessel_296', 'hepaticvessel_297', 'hepaticvessel_299', 'hepaticvessel_300', 'hepaticvessel_303', 'hepaticvessel_305', 'hepaticvessel_308', 'hepaticvessel_309', 'hepaticvessel_314', 'hepaticvessel_319', 'hepaticvessel_320', 'hepaticvessel_321', 'hepaticvessel_323', 'hepaticvessel_324', 'hepaticvessel_325', 'hepaticvessel_327', 'hepaticvessel_329', 'hepaticvessel_330', 'hepaticvessel_332', 'hepaticvessel_333', 'hepaticvessel_335', 'hepaticvessel_336', 'hepaticvessel_340', 'hepaticvessel_341', 'hepaticvessel_344', 'hepaticvessel_345', 'hepaticvessel_348', 'hepaticvessel_349', 'hepaticvessel_350', 'hepaticvessel_356', 'hepaticvessel_358', 'hepaticvessel_359', 'hepaticvessel_361', 'hepaticvessel_362', 'hepaticvessel_363', 'hepaticvessel_368', 'hepaticvessel_369', 'hepaticvessel_371', 'hepaticvessel_372', 'hepaticvessel_373', 'hepaticvessel_374', 'hepaticvessel_375', 'hepaticvessel_378', 'hepaticvessel_379', 'hepaticvessel_383', 'hepaticvessel_384', 'hepaticvessel_386', 'hepaticvessel_388', 'hepaticvessel_389', 'hepaticvessel_391', 'hepaticvessel_393', 'hepaticvessel_396', 'hepaticvessel_397', 'hepaticvessel_400', 'hepaticvessel_401', 'hepaticvessel_402', 'hepaticvessel_404', 'hepaticvessel_406', 'hepaticvessel_407', 'hepaticvessel_408', 'hepaticvessel_409', 'hepaticvessel_411', 'hepaticvessel_412', 'hepaticvessel_416', 'hepaticvessel_422', 'hepaticvessel_424', 'hepaticvessel_425', 'hepaticvessel_427', 'hepaticvessel_429', 'hepaticvessel_432', 'hepaticvessel_434', 'hepaticvessel_437', 'hepaticvessel_438', 'hepaticvessel_440', 'hepaticvessel_441', 'hepaticvessel_443', 'hepaticvessel_444', 'hepaticvessel_445', 'hepaticvessel_446', 'hepaticvessel_447', 'hepaticvessel_453', 'hepaticvessel_454', 'hepaticvessel_455', 'hepaticvessel_456', 'hepaticvessel_458']) 
+2022-05-27 13:10:18.506382: VALIDATION KEYS:
+ odict_keys(['hepaticvessel_002', 'hepaticvessel_004', 'hepaticvessel_007', 'hepaticvessel_022', 'hepaticvessel_039', 'hepaticvessel_040', 'hepaticvessel_042', 'hepaticvessel_046', 'hepaticvessel_051', 'hepaticvessel_068', 'hepaticvessel_078', 'hepaticvessel_081', 'hepaticvessel_086', 'hepaticvessel_091', 'hepaticvessel_097', 'hepaticvessel_116', 'hepaticvessel_117', 'hepaticvessel_125', 'hepaticvessel_128', 'hepaticvessel_137', 'hepaticvessel_138', 'hepaticvessel_139', 'hepaticvessel_140', 'hepaticvessel_141', 'hepaticvessel_144', 'hepaticvessel_151', 'hepaticvessel_160', 'hepaticvessel_172', 'hepaticvessel_178', 'hepaticvessel_184', 'hepaticvessel_193', 'hepaticvessel_197', 'hepaticvessel_201', 'hepaticvessel_203', 'hepaticvessel_209', 'hepaticvessel_214', 'hepaticvessel_229', 'hepaticvessel_233', 'hepaticvessel_236', 'hepaticvessel_238', 'hepaticvessel_263', 'hepaticvessel_265', 'hepaticvessel_268', 'hepaticvessel_280', 'hepaticvessel_286', 'hepaticvessel_289', 'hepaticvessel_307', 'hepaticvessel_318', 'hepaticvessel_322', 'hepaticvessel_338', 'hepaticvessel_353', 'hepaticvessel_367', 'hepaticvessel_382', 'hepaticvessel_385', 'hepaticvessel_398', 'hepaticvessel_419', 'hepaticvessel_420', 'hepaticvessel_423', 'hepaticvessel_431', 'hepaticvessel_433', 'hepaticvessel_442']) 
+2022-05-27 13:10:26.430544: loading checkpoint /opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_2/model_latest.model train= True 
+2022-05-27 13:10:27.457920: lr: 5.1e-05 
+2022-05-27 13:10:40.519020: 
+epoch:  93 
+2022-05-27 13:18:19.066530: train loss : 0.0870 
+2022-05-27 13:18:43.959040: validation loss: 0.1025 
+2022-05-27 13:18:43.959574: Average global foreground Dice: [0.6451, 0.8471] 
+2022-05-27 13:18:43.959681: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 13:18:44.396659: lr: 5.1e-05 
+2022-05-27 13:18:44.396825: saving scheduled checkpoint file... 
+2022-05-27 13:18:44.469290: saving checkpoint... 
+2022-05-27 13:18:45.371220: done, saving took 0.97 seconds 
+2022-05-27 13:18:45.380165: done 
+2022-05-27 13:18:54.655742: This epoch took 494.134558 s
+ 
+2022-05-27 13:18:54.655977: 
+epoch:  94 
+2022-05-27 13:26:21.435696: train loss : 0.0894 
+2022-05-27 13:26:46.319200: validation loss: 0.1118 
+2022-05-27 13:26:46.319757: Average global foreground Dice: [0.641, 0.8675] 
+2022-05-27 13:26:46.319873: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 13:26:46.747636: lr: 4.8e-05 
+2022-05-27 13:26:46.747775: saving scheduled checkpoint file... 
+2022-05-27 13:26:46.807386: saving checkpoint... 
+2022-05-27 13:26:47.676966: done, saving took 0.93 seconds 
+2022-05-27 13:26:47.684530: done 
+2022-05-27 13:26:47.703182: saving checkpoint... 
+2022-05-27 13:26:48.314137: done, saving took 0.63 seconds 
+2022-05-27 13:27:26.059867: This epoch took 511.403738 s
+ 
+2022-05-27 13:27:26.060159: 
+epoch:  95 
+2022-05-27 13:34:52.701140: train loss : 0.0913 
+2022-05-27 13:35:17.599377: validation loss: 0.1196 
+2022-05-27 13:35:17.600002: Average global foreground Dice: [0.6529, 0.8528] 
+2022-05-27 13:35:17.600099: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 13:35:18.026276: lr: 4.5e-05 
+2022-05-27 13:35:18.026423: saving scheduled checkpoint file... 
+2022-05-27 13:35:18.085472: saving checkpoint... 
+2022-05-27 13:35:18.935143: done, saving took 0.91 seconds 
+2022-05-27 13:35:18.942611: done 
+2022-05-27 13:35:18.959168: saving checkpoint... 
+2022-05-27 13:35:19.768434: done, saving took 0.83 seconds 
+2022-05-27 13:35:42.514996: This epoch took 496.454692 s
+ 
+2022-05-27 13:35:42.515265: 
+epoch:  96 
+2022-05-27 13:43:09.050013: train loss : 0.0932 
+2022-05-27 13:43:33.913221: validation loss: 0.1126 
+2022-05-27 13:43:33.913707: Average global foreground Dice: [0.6313, 0.8374] 
+2022-05-27 13:43:33.913803: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 13:43:34.343753: lr: 4.2e-05 
+2022-05-27 13:43:34.343904: saving scheduled checkpoint file... 
+2022-05-27 13:43:34.402983: saving checkpoint... 
+2022-05-27 13:43:35.347799: done, saving took 1.00 seconds 
+2022-05-27 13:43:35.356315: done 
+2022-05-27 13:44:00.653875: This epoch took 498.138493 s
+ 
+2022-05-27 13:44:00.654253: 
+epoch:  97 
+2022-05-27 13:51:27.551327: train loss : 0.0927 
+2022-05-27 13:51:52.410709: validation loss: 0.1108 
+2022-05-27 13:51:52.411217: Average global foreground Dice: [0.6421, 0.8218] 
+2022-05-27 13:51:52.411311: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 13:51:52.840672: lr: 3.9e-05 
+2022-05-27 13:51:52.840822: saving scheduled checkpoint file... 
+2022-05-27 13:51:52.900112: saving checkpoint... 
+2022-05-27 13:51:53.833024: done, saving took 0.99 seconds 
+2022-05-27 13:51:53.841214: done 
+2022-05-27 13:52:29.609810: This epoch took 508.955400 s
+ 
+2022-05-27 13:52:29.610171: 
+epoch:  98 
+2022-05-27 13:59:56.230547: train loss : 0.0936 
+2022-05-27 14:00:21.135344: validation loss: 0.1229 
+2022-05-27 14:00:21.135980: Average global foreground Dice: [0.638, 0.8422] 
+2022-05-27 14:00:21.136074: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 14:00:21.565876: lr: 3.6e-05 
+2022-05-27 14:00:21.566022: saving scheduled checkpoint file... 
+2022-05-27 14:00:21.625840: saving checkpoint... 
+2022-05-27 14:00:22.549603: done, saving took 0.98 seconds 
+2022-05-27 14:00:22.557997: done 
+2022-05-27 14:00:43.473881: This epoch took 493.863556 s
+ 
+2022-05-27 14:00:43.474148: 
+epoch:  99 
+2022-05-27 14:08:10.037102: train loss : 0.0854 
+2022-05-27 14:08:34.928749: validation loss: 0.1092 
+2022-05-27 14:08:34.929256: Average global foreground Dice: [0.6473, 0.8002] 
+2022-05-27 14:08:34.929346: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 14:08:35.356624: lr: 3.4e-05 
+2022-05-27 14:08:35.356843: saving scheduled checkpoint file... 
+2022-05-27 14:08:35.416116: saving checkpoint... 
+2022-05-27 14:08:36.249519: done, saving took 0.89 seconds 
+2022-05-27 14:08:36.258090: done 
+2022-05-27 14:09:10.303372: This epoch took 506.829124 s
+ 
+2022-05-27 14:09:10.303654: 
+epoch:  100 
+2022-05-27 14:16:37.200175: train loss : 0.0900 
+2022-05-27 14:17:02.112207: validation loss: 0.1010 
+2022-05-27 14:17:02.112753: Average global foreground Dice: [0.6444, 0.8565] 
+2022-05-27 14:17:02.112850: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 14:17:02.541251: lr: 3.1e-05 
+2022-05-27 14:17:02.541404: saving scheduled checkpoint file... 
+2022-05-27 14:17:02.601057: saving checkpoint... 
+2022-05-27 14:17:03.433770: done, saving took 0.89 seconds 
+2022-05-27 14:17:03.441698: done 
+2022-05-27 14:17:57.787756: This epoch took 527.483940 s
+ 
+2022-05-27 14:17:57.788044: 
+epoch:  101 
+2022-05-27 14:25:24.351850: train loss : 0.0935 
+2022-05-27 14:25:49.254310: validation loss: 0.1036 
+2022-05-27 14:25:49.254805: Average global foreground Dice: [0.6424, 0.8513] 
+2022-05-27 14:25:49.254899: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 14:25:49.694954: lr: 2.9e-05 
+2022-05-27 14:25:49.695111: saving scheduled checkpoint file... 
+2022-05-27 14:25:49.754292: saving checkpoint... 
+2022-05-27 14:25:50.732259: done, saving took 1.04 seconds 
+2022-05-27 14:25:50.740791: done 
+2022-05-27 14:26:16.473124: This epoch took 498.684963 s
+ 
+2022-05-27 14:26:16.473398: 
+epoch:  102 
+2022-05-27 14:33:43.073627: train loss : 0.0956 
+2022-05-27 14:34:07.982656: validation loss: 0.1130 
+2022-05-27 14:34:07.983187: Average global foreground Dice: [0.6481, 0.8496] 
+2022-05-27 14:34:07.983282: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 14:34:08.414323: lr: 2.6e-05 
+2022-05-27 14:34:08.414468: saving scheduled checkpoint file... 
+2022-05-27 14:34:08.474471: saving checkpoint... 
+2022-05-27 14:34:09.396829: done, saving took 0.98 seconds 
+2022-05-27 14:34:09.405622: done 
+2022-05-27 14:34:25.851990: This epoch took 489.378482 s
+ 
+2022-05-27 14:34:25.852248: 
+epoch:  103 
+2022-05-27 14:41:52.325432: train loss : 0.0935 
+2022-05-27 14:42:17.229959: validation loss: 0.1222 
+2022-05-27 14:42:17.230487: Average global foreground Dice: [0.6314, 0.8707] 
+2022-05-27 14:42:17.230581: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 14:42:17.663993: lr: 2.4e-05 
+2022-05-27 14:42:17.664150: saving scheduled checkpoint file... 
+2022-05-27 14:42:17.723634: saving checkpoint... 
+2022-05-27 14:42:18.676801: done, saving took 1.01 seconds 
+2022-05-27 14:42:18.685125: done 
+2022-05-27 14:42:38.120981: This epoch took 492.268629 s
+ 
+2022-05-27 14:42:38.121239: 
+epoch:  104 
+2022-05-27 14:50:05.018342: train loss : 0.0996 
+2022-05-27 14:50:29.892182: validation loss: 0.1046 
+2022-05-27 14:50:29.892713: Average global foreground Dice: [0.6237, 0.8687] 
+2022-05-27 14:50:29.892809: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 14:50:30.323799: lr: 2.2e-05 
+2022-05-27 14:50:30.323951: saving scheduled checkpoint file... 
+2022-05-27 14:50:30.383751: saving checkpoint... 
+2022-05-27 14:50:31.352848: done, saving took 1.03 seconds 
+2022-05-27 14:50:31.362734: done 
+2022-05-27 14:50:54.676013: This epoch took 496.554671 s
+ 
+2022-05-27 14:50:54.676284: 
+epoch:  105 
+2022-05-27 14:58:21.394512: train loss : 0.0953 
+2022-05-27 14:58:46.286985: validation loss: 0.1110 
+2022-05-27 14:58:46.287517: Average global foreground Dice: [0.6502, 0.8511] 
+2022-05-27 14:58:46.287616: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 14:58:46.716838: lr: 2e-05 
+2022-05-27 14:58:46.716985: saving scheduled checkpoint file... 
+2022-05-27 14:58:46.776605: saving checkpoint... 
+2022-05-27 14:58:47.710517: done, saving took 0.99 seconds 
+2022-05-27 14:58:47.719355: done 
+2022-05-27 14:59:14.278253: This epoch took 499.601860 s
+ 
+2022-05-27 14:59:14.278527: 
+epoch:  106 
+2022-05-27 15:06:41.096495: train loss : 0.0979 
+2022-05-27 15:07:06.004962: validation loss: 0.1237 
+2022-05-27 15:07:06.005507: Average global foreground Dice: [0.6348, 0.8571] 
+2022-05-27 15:07:06.005607: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 15:07:06.434954: lr: 1.8e-05 
+2022-05-27 15:07:06.435095: saving scheduled checkpoint file... 
+2022-05-27 15:07:06.494618: saving checkpoint... 
+2022-05-27 15:07:07.418174: done, saving took 0.98 seconds 
+2022-05-27 15:07:07.426654: done 
+2022-05-27 15:07:32.592615: This epoch took 498.313931 s
+ 
+2022-05-27 15:07:32.592895: 
+epoch:  107 
+2022-05-27 15:14:59.477870: train loss : 0.0961 
+2022-05-27 15:15:24.370386: validation loss: 0.1084 
+2022-05-27 15:15:24.370879: Average global foreground Dice: [0.6438, 0.8656] 
+2022-05-27 15:15:24.370977: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 15:15:24.805041: lr: 1.6e-05 
+2022-05-27 15:15:24.805193: saving scheduled checkpoint file... 
+2022-05-27 15:15:24.865577: saving checkpoint... 
+2022-05-27 15:15:25.822587: done, saving took 1.02 seconds 
+2022-05-27 15:15:25.831582: done 
+2022-05-27 15:15:58.220585: This epoch took 505.627577 s
+ 
+2022-05-27 15:15:58.220893: 
+epoch:  108 
+2022-05-27 15:23:25.215914: train loss : 0.0928 
+2022-05-27 15:23:50.114080: validation loss: 0.1173 
+2022-05-27 15:23:50.114579: Average global foreground Dice: [0.6301, 0.816] 
+2022-05-27 15:23:50.114680: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 15:23:50.550763: lr: 1.4e-05 
+2022-05-27 15:23:50.550915: saving scheduled checkpoint file... 
+2022-05-27 15:23:50.610373: saving checkpoint... 
+2022-05-27 15:23:51.546987: done, saving took 1.00 seconds 
+2022-05-27 15:23:51.555466: done 
+2022-05-27 15:24:08.393326: This epoch took 490.172274 s
+ 
+2022-05-27 15:24:08.393613: 
+epoch:  109 
+2022-05-27 15:31:35.154878: train loss : 0.1009 
+2022-05-27 15:32:00.058503: validation loss: 0.1079 
+2022-05-27 15:32:00.059041: Average global foreground Dice: [0.636, 0.8623] 
+2022-05-27 15:32:00.059134: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 15:32:00.486300: lr: 1.2e-05 
+2022-05-27 15:32:00.486473: saving scheduled checkpoint file... 
+2022-05-27 15:32:00.545723: saving checkpoint... 
+2022-05-27 15:32:01.397580: done, saving took 0.91 seconds 
+2022-05-27 15:32:01.404934: done 
+2022-05-27 15:32:21.368201: This epoch took 492.974409 s
+ 
+2022-05-27 15:32:21.368479: 
+epoch:  110 
+2022-05-27 15:39:48.174214: train loss : 0.0919 
+2022-05-27 15:40:13.072838: validation loss: 0.1280 
+2022-05-27 15:40:13.073381: Average global foreground Dice: [0.6295, 0.8385] 
+2022-05-27 15:40:13.073514: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 15:40:13.505037: lr: 1.1e-05 
+2022-05-27 15:40:13.505179: saving scheduled checkpoint file... 
+2022-05-27 15:40:13.565659: saving checkpoint... 
+2022-05-27 15:40:14.511063: done, saving took 1.01 seconds 
+2022-05-27 15:40:14.519791: done 
+2022-05-27 15:40:40.466559: This epoch took 499.097975 s
+ 
+2022-05-27 15:40:40.466837: 
+epoch:  111 
+2022-05-27 15:48:07.168368: train loss : 0.0917 
+2022-05-27 15:48:32.059286: validation loss: 0.1096 
+2022-05-27 15:48:32.059820: Average global foreground Dice: [0.6401, 0.8737] 
+2022-05-27 15:48:32.059914: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 15:48:32.492384: lr: 9e-06 
+2022-05-27 15:48:32.492533: saving scheduled checkpoint file... 
+2022-05-27 15:48:32.551532: saving checkpoint... 
+2022-05-27 15:48:33.358493: done, saving took 0.87 seconds 
+2022-05-27 15:48:33.366428: done 
+2022-05-27 15:48:51.182449: This epoch took 490.715497 s
+ 
+2022-05-27 15:48:51.182729: 
+epoch:  112 
+2022-05-27 15:56:17.956728: train loss : 0.0903 
+2022-05-27 15:56:42.918421: validation loss: 0.1119 
+2022-05-27 15:56:42.918951: Average global foreground Dice: [0.6434, 0.8565] 
+2022-05-27 15:56:42.919049: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 15:56:43.347725: lr: 8e-06 
+2022-05-27 15:56:43.347867: saving scheduled checkpoint file... 
+2022-05-27 15:56:43.407545: saving checkpoint... 
+2022-05-27 15:56:44.373216: done, saving took 1.03 seconds 
+2022-05-27 15:56:44.381617: done 
+2022-05-27 15:57:03.970626: This epoch took 492.787778 s
+ 
+2022-05-27 15:57:03.970927: 
+epoch:  113 
+2022-05-27 16:04:30.841586: train loss : 0.0979 
+2022-05-27 16:04:55.737650: validation loss: 0.0937 
+2022-05-27 16:04:55.738170: Average global foreground Dice: [0.657, 0.8736] 
+2022-05-27 16:04:55.738266: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 16:04:56.171224: lr: 7e-06 
+2022-05-27 16:04:56.171370: saving scheduled checkpoint file... 
+2022-05-27 16:04:56.231713: saving checkpoint... 
+2022-05-27 16:04:57.012997: done, saving took 0.84 seconds 
+2022-05-27 16:04:57.021365: done 
+2022-05-27 16:05:25.835487: This epoch took 501.864450 s
+ 
+2022-05-27 16:05:25.835761: 
+epoch:  114 
+2022-05-27 16:12:52.703648: train loss : 0.0896 
+2022-05-27 16:13:17.592824: validation loss: 0.1224 
+2022-05-27 16:13:17.593349: Average global foreground Dice: [0.6334, 0.8565] 
+2022-05-27 16:13:17.593441: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 16:13:18.024523: lr: 6e-06 
+2022-05-27 16:13:18.024674: saving scheduled checkpoint file... 
+2022-05-27 16:13:18.084560: saving checkpoint... 
+2022-05-27 16:13:19.021394: done, saving took 1.00 seconds 
+2022-05-27 16:13:19.030232: done 
+2022-05-27 16:13:45.562142: This epoch took 499.726276 s
+ 
+2022-05-27 16:13:45.562407: 
+epoch:  115 
+2022-05-27 16:21:12.164934: train loss : 0.0956 
+2022-05-27 16:21:37.102291: validation loss: 0.1189 
+2022-05-27 16:21:37.102772: Average global foreground Dice: [0.6288, 0.8582] 
+2022-05-27 16:21:37.102869: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 16:21:37.541187: lr: 5e-06 
+2022-05-27 16:21:37.541336: saving scheduled checkpoint file... 
+2022-05-27 16:21:37.600466: saving checkpoint... 
+2022-05-27 16:21:38.472806: done, saving took 0.93 seconds 
+2022-05-27 16:21:38.481578: done 
+2022-05-27 16:22:13.659984: This epoch took 508.097450 s
+ 
+2022-05-27 16:22:13.660254: 
+epoch:  116 
+2022-05-27 16:29:40.276144: train loss : 0.1007 
+2022-05-27 16:30:05.173776: validation loss: 0.1417 
+2022-05-27 16:30:05.174305: Average global foreground Dice: [0.6301, 0.8305] 
+2022-05-27 16:30:05.174398: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 16:30:05.606249: lr: 4e-06 
+2022-05-27 16:30:05.606395: saving scheduled checkpoint file... 
+2022-05-27 16:30:05.666514: saving checkpoint... 
+2022-05-27 16:30:06.625474: done, saving took 1.02 seconds 
+2022-05-27 16:30:06.634530: done 
+2022-05-27 16:30:30.101359: This epoch took 496.440919 s
+ 
+2022-05-27 16:30:30.101636: 
+epoch:  117 
+2022-05-27 16:37:57.590614: train loss : 0.0959 
+2022-05-27 16:38:22.525457: validation loss: 0.1067 
+2022-05-27 16:38:22.525951: Average global foreground Dice: [0.6493, 0.8503] 
+2022-05-27 16:38:22.526044: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 16:38:22.957292: lr: 3e-06 
+2022-05-27 16:38:22.957438: saving scheduled checkpoint file... 
+2022-05-27 16:38:23.018775: saving checkpoint... 
+2022-05-27 16:38:23.957217: done, saving took 1.00 seconds 
+2022-05-27 16:38:23.966562: done 
+2022-05-27 16:38:55.393576: This epoch took 505.291824 s
+ 
+2022-05-27 16:38:55.393852: 
+epoch:  118 
+2022-05-27 16:46:22.528802: train loss : 0.0921 
+2022-05-27 16:46:47.408857: validation loss: 0.1323 
+2022-05-27 16:46:47.409426: Average global foreground Dice: [0.6483, 0.8195] 
+2022-05-27 16:46:47.409568: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 16:46:47.844984: lr: 2e-06 
+2022-05-27 16:46:47.845133: saving scheduled checkpoint file... 
+2022-05-27 16:46:47.905187: saving checkpoint... 
+2022-05-27 16:46:48.888550: done, saving took 1.04 seconds 
+2022-05-27 16:46:48.896803: done 
+2022-05-27 16:47:04.766850: This epoch took 489.372885 s
+ 
+2022-05-27 16:47:04.767108: 
+epoch:  119 
+2022-05-27 16:54:31.386301: train loss : 0.0948 
+2022-05-27 16:54:56.287969: validation loss: 0.1138 
+2022-05-27 16:54:56.288428: Average global foreground Dice: [0.6351, 0.8493] 
+2022-05-27 16:54:56.288521: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 16:54:56.721251: lr: 1e-06 
+2022-05-27 16:54:56.721396: saving scheduled checkpoint file... 
+2022-05-27 16:54:56.780962: saving checkpoint... 
+2022-05-27 16:54:57.715408: done, saving took 0.99 seconds 
+2022-05-27 16:54:57.724087: done 
+2022-05-27 16:55:16.704445: This epoch took 491.937224 s
+ 
+2022-05-27 16:55:16.704731: 
+epoch:  120 
+2022-05-27 17:02:43.409763: train loss : 0.0915 
+2022-05-27 17:03:08.292662: validation loss: 0.1181 
+2022-05-27 17:03:08.293314: Average global foreground Dice: [0.6352, 0.8234] 
+2022-05-27 17:03:08.293408: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 17:03:08.725948: lr: 1e-06 
+2022-05-27 17:03:08.726123: saving scheduled checkpoint file... 
+2022-05-27 17:03:08.785564: saving checkpoint... 
+2022-05-27 17:03:09.732177: done, saving took 1.01 seconds 
+2022-05-27 17:03:09.740957: done 
+2022-05-27 17:03:27.522784: This epoch took 490.817944 s
+ 
+2022-05-27 17:03:27.523069: 
+epoch:  121 
+2022-05-27 17:10:54.362101: train loss : 0.0937 
+2022-05-27 17:11:19.299269: validation loss: 0.1193 
+2022-05-27 17:11:19.299772: Average global foreground Dice: [0.6327, 0.8579] 
+2022-05-27 17:11:19.299869: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 17:11:19.733926: lr: 1e-06 
+2022-05-27 17:11:19.734084: saving scheduled checkpoint file... 
+2022-05-27 17:11:19.793018: saving checkpoint... 
+2022-05-27 17:11:20.722040: done, saving took 0.99 seconds 
+2022-05-27 17:11:20.730273: done 
+2022-05-27 17:11:37.817078: This epoch took 490.293847 s
+ 
+2022-05-27 17:11:37.817355: 
+epoch:  122 
+2022-05-27 17:19:05.053128: train loss : 0.0944 
+2022-05-27 17:19:29.970866: validation loss: 0.1027 
+2022-05-27 17:19:29.971473: Average global foreground Dice: [0.6488, 0.8721] 
+2022-05-27 17:19:29.971600: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 17:19:30.402567: lr: 0.0 
+2022-05-27 17:19:30.402712: saving scheduled checkpoint file... 
+2022-05-27 17:19:30.461819: saving checkpoint... 
+2022-05-27 17:19:31.390275: done, saving took 0.99 seconds 
+2022-05-27 17:19:31.398940: done 
+2022-05-27 17:20:04.272370: This epoch took 506.454912 s
+ 
+2022-05-27 17:20:04.272653: 
+epoch:  123 
+2022-05-27 17:27:30.965046: train loss : 0.1005 
+2022-05-27 17:27:55.886603: validation loss: 0.0966 
+2022-05-27 17:27:55.887138: Average global foreground Dice: [0.6397, 0.8297] 
+2022-05-27 17:27:55.887242: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 17:27:56.322790: lr: 0.0 
+2022-05-27 17:27:56.322941: saving scheduled checkpoint file... 
+2022-05-27 17:27:56.382082: saving checkpoint... 
+2022-05-27 17:27:57.314850: done, saving took 0.99 seconds 
+2022-05-27 17:27:57.323086: done 
+2022-05-27 17:28:33.274993: This epoch took 509.002238 s
+ 
+2022-05-27 17:28:33.275266: 
+epoch:  124 
+2022-05-27 17:35:59.980655: train loss : 0.0991 
+2022-05-27 17:36:24.884819: validation loss: 0.1154 
+2022-05-27 17:36:24.885308: Average global foreground Dice: [0.6377, 0.8568] 
+2022-05-27 17:36:24.885403: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-27 17:36:25.316325: lr: 0.0 
+2022-05-27 17:36:25.316481: saving scheduled checkpoint file... 
+2022-05-27 17:36:25.376010: saving checkpoint... 
+2022-05-27 17:36:26.311341: done, saving took 0.99 seconds 
+2022-05-27 17:36:26.320225: done 
+2022-05-27 17:36:45.600882: This epoch took 492.325458 s
+ 
+2022-05-27 17:36:45.648484: saving checkpoint... 
+2022-05-27 17:36:46.277766: done, saving took 0.68 seconds 

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/debug.json

@@ -0,0 +1,106 @@
+{
+    "all_tr_losses": "[]",
+    "all_val_eval_metrics": "[]",
+    "all_val_losses": "[]",
+    "all_val_losses_tr_mode": "[]",
+    "also_val_in_tr_mode": "False",
+    "amp_grad_scaler": "None",
+    "args": "Namespace(batch_size=2, config='./configs/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32.yaml', continue_training=False, crop_size=[64, 192, 192], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='3', fp32=False, hdfs_base='GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=125, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task008_HepaticVessel', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)",
+    "base_num_features": "32",
+    "basic_generator_patch_size": "[178 308 233]",
+    "batch_dice": "True",
+    "batch_size": "2",
+    "best_MA_tr_loss_for_patience": "None",
+    "best_epoch_based_on_MA_tr_loss": "None",
+    "best_val_eval_criterion_MA": "None",
+    "classes": "[1, 2]",
+    "conv_per_stage": "2",
+    "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-0.5235987755982988, 0.5235987755982988), 'rotation_y': (-0.5235987755982988, 0.5235987755982988), 'rotation_z': (-0.5235987755982988, 0.5235987755982988), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([ 64, 192, 192])}",
+    "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel",
+    "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]",
+    "deterministic": "False",
+    "disable_ds": "True",
+    "distribute_batch_size": "False",
+    "dl_tr": "<dataset_loading.DataLoader3D object at 0x7f3dbdb4cef0>",
+    "dl_val": "<dataset_loading.DataLoader3D object at 0x7f3dbdbd5c88>",
+    "do_dummy_2D_aug": "False",
+    "ds_loss_weights": "[1. 0. 0. 0. 0.]",
+    "epoch": "0",
+    "experiment_name": "nnUNetTrainerV2_DDP",
+    "find_zero_weight_decay": "False",
+    "fold": "3",
+    "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetData_plans_v2.1_stage1",
+    "fp16": "True",
+    "global_batch_size": "16",
+    "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/gt_segmentations",
+    "inference_pad_border_mode": "constant",
+    "inference_pad_kwargs": "{'constant_values': 0}",
+    "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl', 3, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', '/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel', True, 1, True, False, False, True)",
+    "initial_lr": "0.0003",
+    "input_size": "[64, 192, 192]",
+    "is_spatial_aug_only": "False",
+    "is_ssl": "False",
+    "layer_decay": "1.0",
+    "local_rank": "0",
+    "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/training_log_2022_4_10_10_20_37.txt",
+    "loss": "None",
+    "lr_scheduler": "<optimizers.lr_scheduler.LinearWarmupCosineAnnealingLR object at 0x7f3d63449e80>",
+    "lr_scheduler_eps": "0.001",
+    "lr_scheduler_name": "warmup_cosine",
+    "lr_scheduler_patience": "30",
+    "lr_threshold": "1e-06",
+    "max_num_epochs": "125",
+    "min_lr": "0",
+    "min_region_size_per_class": "None",
+    "min_size_per_class": "None",
+    "model": "Generic_TransUNet_max_ppbp",
+    "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}",
+    "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]",
+    "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]]",
+    "net_pool_per_axis": "[4, 5, 5]",
+    "normalization_schemes": "OrderedDict([(0, 'CT')])",
+    "num_batches_per_epoch": "250",
+    "num_classes": "3",
+    "num_input_channels": "1",
+    "num_val_batches_per_epoch": "50",
+    "online_eval_fn": "[]",
+    "online_eval_foreground_dc": "[]",
+    "online_eval_fp": "[]",
+    "online_eval_tp": "[]",
+    "only_keep_largest_connected_component": "None",
+    "optim_name": "adamw",
+    "optimizer": "AdamW (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    eps: 1e-08\n    initial_lr: 0.0003\n    lr: 0.0\n    weight_decay: 3e-05\n)",
+    "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3",
+    "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32",
+    "oversample_foreground_percent": "0.0",
+    "pad_all_sides": "None",
+    "patch_size": "[ 64 192 192]",
+    "patience": "50",
+    "pin_memory": "True",
+    "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl",
+    "reclip": "None",
+    "regions_class_order": "None",
+    "resume": "auto",
+    "save_best_checkpoint": "True",
+    "save_every": "1",
+    "save_final_checkpoint": "True",
+    "save_intermediate_checkpoints": "True",
+    "save_latest_only": "True",
+    "stage": "1",
+    "threeD": "True",
+    "tr_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7f3db4244ac8>",
+    "train_loss_MA": "None",
+    "train_loss_MA_alpha": "0.93",
+    "train_loss_MA_eps": "0.0005",
+    "transpose_backward": "[0, 1, 2]",
+    "transpose_forward": "[0, 1, 2]",
+    "unpack_data": "True",
+    "use_mask_for_norm": "OrderedDict([(0, False)])",
+    "use_progress_bar": "False",
+    "val_eval_criterion_MA": "None",
+    "val_eval_criterion_alpha": "0.9",
+    "val_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7f3db02011d0>",
+    "warmup_epochs": "10",
+    "was_initialized": "True",
+    "weight_decay": "3e-05"
+}

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_best.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e519551535b456c82bf4f72ec5616bc6adc12612e71cab3cfc05edd9ffee190
+size 496529529

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_best.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ba3cdd78f78160c349ef0250bdf2abd5e4b8a5ea11911f42c5fdba0bfad77a6
+size 188314

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_final_checkpoint.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be1b43eaf3027ebfdf6dc5fa57ffcff2ae50d8c78a972165f3f36f67320d6618
+size 496529529

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_final_checkpoint.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ba3cdd78f78160c349ef0250bdf2abd5e4b8a5ea11911f42c5fdba0bfad77a6
+size 188314

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_latest.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bc174f6585d0adbbdb6ae9fb91c8b11202dd6f22f20526debe0ac4490d16c01
+size 496529529

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/model_latest.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ba3cdd78f78160c349ef0250bdf2abd5e4b8a5ea11911f42c5fdba0bfad77a6
+size 188314

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_3/training_log_2022_4_10_10_20_37.txt

@@ -0,0 +1,1787 @@
+Starting... 
+2022-04-10 10:20:37.083468: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/splits_final.pkl 
+2022-04-10 10:20:37.084042: The split file contains 5 splits. 
+2022-04-10 10:20:37.084095: Desired fold for training: 3 
+2022-04-10 10:20:37.084203: This split has 243 training and 60 validation cases. 
+2022-04-10 10:22:54.201807: TRAINING KEYS:
+ odict_keys(['hepaticvessel_002', 'hepaticvessel_004', 'hepaticvessel_007', 'hepaticvessel_008', 'hepaticvessel_016', 'hepaticvessel_018', 'hepaticvessel_019', 'hepaticvessel_020', 'hepaticvessel_022', 'hepaticvessel_023', 'hepaticvessel_025', 'hepaticvessel_026', 'hepaticvessel_027', 'hepaticvessel_028', 'hepaticvessel_029', 'hepaticvessel_030', 'hepaticvessel_032', 'hepaticvessel_033', 'hepaticvessel_039', 'hepaticvessel_040', 'hepaticvessel_042', 'hepaticvessel_044', 'hepaticvessel_046', 'hepaticvessel_049', 'hepaticvessel_050', 'hepaticvessel_051', 'hepaticvessel_053', 'hepaticvessel_057', 'hepaticvessel_058', 'hepaticvessel_059', 'hepaticvessel_062', 'hepaticvessel_065', 'hepaticvessel_066', 'hepaticvessel_067', 'hepaticvessel_068', 'hepaticvessel_070', 'hepaticvessel_071', 'hepaticvessel_075', 'hepaticvessel_077', 'hepaticvessel_078', 'hepaticvessel_079', 'hepaticvessel_080', 'hepaticvessel_081', 'hepaticvessel_082', 'hepaticvessel_083', 'hepaticvessel_084', 'hepaticvessel_085', 'hepaticvessel_086', 'hepaticvessel_087', 'hepaticvessel_088', 'hepaticvessel_089', 'hepaticvessel_091', 'hepaticvessel_092', 'hepaticvessel_094', 'hepaticvessel_097', 'hepaticvessel_098', 'hepaticvessel_100', 'hepaticvessel_103', 'hepaticvessel_104', 'hepaticvessel_108', 'hepaticvessel_110', 'hepaticvessel_111', 'hepaticvessel_112', 'hepaticvessel_113', 'hepaticvessel_115', 'hepaticvessel_116', 'hepaticvessel_117', 'hepaticvessel_119', 'hepaticvessel_121', 'hepaticvessel_123', 'hepaticvessel_124', 'hepaticvessel_125', 'hepaticvessel_127', 'hepaticvessel_128', 'hepaticvessel_129', 'hepaticvessel_131', 'hepaticvessel_132', 'hepaticvessel_136', 'hepaticvessel_137', 'hepaticvessel_138', 'hepaticvessel_139', 'hepaticvessel_140', 'hepaticvessel_141', 'hepaticvessel_142', 'hepaticvessel_144', 'hepaticvessel_145', 'hepaticvessel_146', 'hepaticvessel_147', 'hepaticvessel_150', 'hepaticvessel_151', 'hepaticvessel_154', 'hepaticvessel_157', 'hepaticvessel_158', 'hepaticvessel_159', 'hepaticvessel_160', 'hepaticvessel_161', 'hepaticvessel_164', 'hepaticvessel_165', 'hepaticvessel_167', 'hepaticvessel_172', 'hepaticvessel_173', 'hepaticvessel_174', 'hepaticvessel_175', 'hepaticvessel_178', 'hepaticvessel_181', 'hepaticvessel_184', 'hepaticvessel_185', 'hepaticvessel_186', 'hepaticvessel_189', 'hepaticvessel_192', 'hepaticvessel_193', 'hepaticvessel_194', 'hepaticvessel_195', 'hepaticvessel_196', 'hepaticvessel_197', 'hepaticvessel_200', 'hepaticvessel_201', 'hepaticvessel_203', 'hepaticvessel_206', 'hepaticvessel_207', 'hepaticvessel_208', 'hepaticvessel_209', 'hepaticvessel_210', 'hepaticvessel_211', 'hepaticvessel_213', 'hepaticvessel_214', 'hepaticvessel_217', 'hepaticvessel_218', 'hepaticvessel_219', 'hepaticvessel_223', 'hepaticvessel_224', 'hepaticvessel_225', 'hepaticvessel_229', 'hepaticvessel_230', 'hepaticvessel_231', 'hepaticvessel_233', 'hepaticvessel_234', 'hepaticvessel_235', 'hepaticvessel_236', 'hepaticvessel_237', 'hepaticvessel_238', 'hepaticvessel_240', 'hepaticvessel_242', 'hepaticvessel_244', 'hepaticvessel_248', 'hepaticvessel_255', 'hepaticvessel_256', 'hepaticvessel_258', 'hepaticvessel_262', 'hepaticvessel_263', 'hepaticvessel_265', 'hepaticvessel_266', 'hepaticvessel_268', 'hepaticvessel_269', 'hepaticvessel_270', 'hepaticvessel_271', 'hepaticvessel_272', 'hepaticvessel_274', 'hepaticvessel_275', 'hepaticvessel_278', 'hepaticvessel_280', 'hepaticvessel_282', 'hepaticvessel_285', 'hepaticvessel_286', 'hepaticvessel_287', 'hepaticvessel_289', 'hepaticvessel_293', 'hepaticvessel_297', 'hepaticvessel_299', 'hepaticvessel_303', 'hepaticvessel_305', 'hepaticvessel_307', 'hepaticvessel_308', 'hepaticvessel_309', 'hepaticvessel_318', 'hepaticvessel_321', 'hepaticvessel_322', 'hepaticvessel_324', 'hepaticvessel_329', 'hepaticvessel_330', 'hepaticvessel_332', 'hepaticvessel_333', 'hepaticvessel_335', 'hepaticvessel_338', 'hepaticvessel_340', 'hepaticvessel_341', 'hepaticvessel_344', 'hepaticvessel_345', 'hepaticvessel_348', 'hepaticvessel_349', 'hepaticvessel_350', 'hepaticvessel_353', 'hepaticvessel_356', 'hepaticvessel_358', 'hepaticvessel_359', 'hepaticvessel_361', 'hepaticvessel_362', 'hepaticvessel_363', 'hepaticvessel_367', 'hepaticvessel_368', 'hepaticvessel_372', 'hepaticvessel_373', 'hepaticvessel_375', 'hepaticvessel_378', 'hepaticvessel_379', 'hepaticvessel_382', 'hepaticvessel_383', 'hepaticvessel_384', 'hepaticvessel_385', 'hepaticvessel_386', 'hepaticvessel_388', 'hepaticvessel_389', 'hepaticvessel_391', 'hepaticvessel_393', 'hepaticvessel_396', 'hepaticvessel_397', 'hepaticvessel_398', 'hepaticvessel_402', 'hepaticvessel_404', 'hepaticvessel_406', 'hepaticvessel_407', 'hepaticvessel_408', 'hepaticvessel_409', 'hepaticvessel_419', 'hepaticvessel_420', 'hepaticvessel_422', 'hepaticvessel_423', 'hepaticvessel_424', 'hepaticvessel_425', 'hepaticvessel_431', 'hepaticvessel_432', 'hepaticvessel_433', 'hepaticvessel_437', 'hepaticvessel_440', 'hepaticvessel_441', 'hepaticvessel_442', 'hepaticvessel_443', 'hepaticvessel_444', 'hepaticvessel_445', 'hepaticvessel_454', 'hepaticvessel_455', 'hepaticvessel_456', 'hepaticvessel_458']) 
+2022-04-10 10:22:54.202080: VALIDATION KEYS:
+ odict_keys(['hepaticvessel_001', 'hepaticvessel_005', 'hepaticvessel_010', 'hepaticvessel_011', 'hepaticvessel_013', 'hepaticvessel_031', 'hepaticvessel_052', 'hepaticvessel_061', 'hepaticvessel_072', 'hepaticvessel_090', 'hepaticvessel_093', 'hepaticvessel_096', 'hepaticvessel_102', 'hepaticvessel_133', 'hepaticvessel_148', 'hepaticvessel_163', 'hepaticvessel_166', 'hepaticvessel_171', 'hepaticvessel_177', 'hepaticvessel_179', 'hepaticvessel_183', 'hepaticvessel_198', 'hepaticvessel_199', 'hepaticvessel_215', 'hepaticvessel_221', 'hepaticvessel_222', 'hepaticvessel_227', 'hepaticvessel_245', 'hepaticvessel_246', 'hepaticvessel_259', 'hepaticvessel_279', 'hepaticvessel_281', 'hepaticvessel_284', 'hepaticvessel_290', 'hepaticvessel_291', 'hepaticvessel_294', 'hepaticvessel_296', 'hepaticvessel_300', 'hepaticvessel_314', 'hepaticvessel_319', 'hepaticvessel_320', 'hepaticvessel_323', 'hepaticvessel_325', 'hepaticvessel_327', 'hepaticvessel_336', 'hepaticvessel_369', 'hepaticvessel_371', 'hepaticvessel_374', 'hepaticvessel_400', 'hepaticvessel_401', 'hepaticvessel_411', 'hepaticvessel_412', 'hepaticvessel_416', 'hepaticvessel_427', 'hepaticvessel_429', 'hepaticvessel_434', 'hepaticvessel_438', 'hepaticvessel_446', 'hepaticvessel_447', 'hepaticvessel_453']) 
+2022-04-10 10:22:57.898256: lr: 3.3e-05 
+2022-04-10 10:23:11.866811: 
+epoch:  0 
+2022-04-10 10:31:13.615781: train loss : 0.3499 
+2022-04-10 10:31:40.062033: validation loss: 0.2614 
+2022-04-10 10:31:40.062728: Average global foreground Dice: [0.5041, 0.0] 
+2022-04-10 10:31:40.062819: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 10:31:40.381839: lr: 3.3e-05 
+2022-04-10 10:31:40.382035: saving scheduled checkpoint file... 
+2022-04-10 10:31:40.447974: saving checkpoint... 
+2022-04-10 10:31:41.099472: done, saving took 0.72 seconds 
+2022-04-10 10:31:41.109838: done 
+2022-04-10 10:31:51.594410: This epoch took 519.727127 s
+ 
+2022-04-10 10:31:51.594667: 
+epoch:  1 
+2022-04-10 10:39:36.138235: train loss : 0.2302 
+2022-04-10 10:40:02.573066: validation loss: 0.2136 
+2022-04-10 10:40:02.573890: Average global foreground Dice: [0.5772, 0.3434] 
+2022-04-10 10:40:02.573979: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 10:40:02.963822: lr: 6.7e-05 
+2022-04-10 10:40:02.964006: saving scheduled checkpoint file... 
+2022-04-10 10:40:03.019991: saving checkpoint... 
+2022-04-10 10:40:04.009094: done, saving took 1.04 seconds 
+2022-04-10 10:40:04.021043: done 
+2022-04-10 10:40:04.050357: saving checkpoint... 
+2022-04-10 10:40:04.687926: done, saving took 0.67 seconds 
+2022-04-10 10:40:27.874324: This epoch took 516.279555 s
+ 
+2022-04-10 10:40:27.874632: 
+epoch:  2 
+2022-04-10 10:48:11.721261: train loss : 0.1984 
+2022-04-10 10:48:38.134556: validation loss: 0.1919 
+2022-04-10 10:48:38.135210: Average global foreground Dice: [0.6074, 0.3237] 
+2022-04-10 10:48:38.135298: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 10:48:38.620289: lr: 0.0001 
+2022-04-10 10:48:38.620481: saving scheduled checkpoint file... 
+2022-04-10 10:48:38.677144: saving checkpoint... 
+2022-04-10 10:48:39.428774: done, saving took 0.81 seconds 
+2022-04-10 10:48:39.441379: done 
+2022-04-10 10:48:39.474663: saving checkpoint... 
+2022-04-10 10:48:40.445369: done, saving took 1.00 seconds 
+2022-04-10 10:48:59.806780: This epoch took 511.932049 s
+ 
+2022-04-10 10:48:59.807105: 
+epoch:  3 
+2022-04-10 10:56:44.449374: train loss : 0.1867 
+2022-04-10 10:57:10.882951: validation loss: 0.1974 
+2022-04-10 10:57:10.883556: Average global foreground Dice: [0.5963, 0.4081] 
+2022-04-10 10:57:10.883644: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 10:57:11.283034: lr: 0.000133 
+2022-04-10 10:57:11.283279: saving scheduled checkpoint file... 
+2022-04-10 10:57:11.340510: saving checkpoint... 
+2022-04-10 10:57:12.089612: done, saving took 0.81 seconds 
+2022-04-10 10:57:12.101259: done 
+2022-04-10 10:57:12.130748: saving checkpoint... 
+2022-04-10 10:57:13.082391: done, saving took 0.98 seconds 
+2022-04-10 10:57:31.599987: This epoch took 511.792763 s
+ 
+2022-04-10 10:57:31.600280: 
+epoch:  4 
+2022-04-10 11:05:15.644006: train loss : 0.1836 
+2022-04-10 11:05:42.069019: validation loss: 0.1585 
+2022-04-10 11:05:42.069885: Average global foreground Dice: [0.621, 0.5706] 
+2022-04-10 11:05:42.069982: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 11:05:42.485110: lr: 0.000167 
+2022-04-10 11:05:42.485316: saving scheduled checkpoint file... 
+2022-04-10 11:05:42.542745: saving checkpoint... 
+2022-04-10 11:05:43.298563: done, saving took 0.81 seconds 
+2022-04-10 11:05:43.311946: done 
+2022-04-10 11:05:43.345692: saving checkpoint... 
+2022-04-10 11:05:44.108574: done, saving took 0.80 seconds 
+2022-04-10 11:06:04.434891: This epoch took 512.834504 s
+ 
+2022-04-10 11:06:04.435206: 
+epoch:  5 
+2022-04-10 11:13:49.219070: train loss : 0.1647 
+2022-04-10 11:14:15.632681: validation loss: 0.1838 
+2022-04-10 11:14:15.633498: Average global foreground Dice: [0.6195, 0.6222] 
+2022-04-10 11:14:15.633591: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 11:14:16.036102: lr: 0.0002 
+2022-04-10 11:14:16.036308: saving scheduled checkpoint file... 
+2022-04-10 11:14:16.093601: saving checkpoint... 
+2022-04-10 11:14:16.851172: done, saving took 0.81 seconds 
+2022-04-10 11:14:16.863828: done 
+2022-04-10 11:14:16.894076: saving checkpoint... 
+2022-04-10 11:14:17.840830: done, saving took 0.98 seconds 
+2022-04-10 11:14:38.725897: This epoch took 514.290590 s
+ 
+2022-04-10 11:14:38.726236: 
+epoch:  6 
+2022-04-10 11:22:23.072448: train loss : 0.1661 
+2022-04-10 11:22:49.504532: validation loss: 0.1518 
+2022-04-10 11:22:49.505234: Average global foreground Dice: [0.6306, 0.6597] 
+2022-04-10 11:22:49.505338: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 11:22:49.911081: lr: 0.000233 
+2022-04-10 11:22:49.911281: saving scheduled checkpoint file... 
+2022-04-10 11:22:49.968486: saving checkpoint... 
+2022-04-10 11:22:50.876866: done, saving took 0.97 seconds 
+2022-04-10 11:22:50.889108: done 
+2022-04-10 11:22:50.918263: saving checkpoint... 
+2022-04-10 11:22:51.743119: done, saving took 0.85 seconds 
+2022-04-10 11:23:09.660321: This epoch took 510.933973 s
+ 
+2022-04-10 11:23:09.660594: 
+epoch:  7 
+2022-04-10 11:30:53.846514: train loss : 0.1585 
+2022-04-10 11:31:20.323537: validation loss: 0.1795 
+2022-04-10 11:31:20.324148: Average global foreground Dice: [0.6117, 0.6074] 
+2022-04-10 11:31:20.324249: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 11:31:20.814788: lr: 0.000267 
+2022-04-10 11:31:20.814977: saving scheduled checkpoint file... 
+2022-04-10 11:31:20.870976: saving checkpoint... 
+2022-04-10 11:31:21.903019: done, saving took 1.09 seconds 
+2022-04-10 11:31:21.915514: done 
+2022-04-10 11:31:21.948765: saving checkpoint... 
+2022-04-10 11:31:22.943116: done, saving took 1.03 seconds 
+2022-04-10 11:31:40.452112: This epoch took 510.791406 s
+ 
+2022-04-10 11:31:40.452425: 
+epoch:  8 
+2022-04-10 11:39:24.678682: train loss : 0.1479 
+2022-04-10 11:39:51.096808: validation loss: 0.1400 
+2022-04-10 11:39:51.097403: Average global foreground Dice: [0.6304, 0.6577] 
+2022-04-10 11:39:51.097494: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 11:39:51.500313: lr: 0.0003 
+2022-04-10 11:39:51.500487: saving scheduled checkpoint file... 
+2022-04-10 11:39:51.557127: saving checkpoint... 
+2022-04-10 11:39:52.527678: done, saving took 1.03 seconds 
+2022-04-10 11:39:52.540079: done 
+2022-04-10 11:39:52.572447: saving checkpoint... 
+2022-04-10 11:39:53.550024: done, saving took 1.01 seconds 
+2022-04-10 11:40:55.656973: This epoch took 555.204448 s
+ 
+2022-04-10 11:40:55.657299: 
+epoch:  9 
+2022-04-10 11:48:39.339222: train loss : 0.1542 
+2022-04-10 11:49:05.771391: validation loss: 0.1571 
+2022-04-10 11:49:05.771981: Average global foreground Dice: [0.6411, 0.7239] 
+2022-04-10 11:49:05.772190: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 11:49:06.158440: lr: 0.0003 
+2022-04-10 11:49:06.158624: saving scheduled checkpoint file... 
+2022-04-10 11:49:06.215422: saving checkpoint... 
+2022-04-10 11:49:06.981601: done, saving took 0.82 seconds 
+2022-04-10 11:49:06.995433: done 
+2022-04-10 11:49:07.029147: saving checkpoint... 
+2022-04-10 11:49:07.802680: done, saving took 0.81 seconds 
+2022-04-10 11:49:26.215940: This epoch took 510.558534 s
+ 
+2022-04-10 11:49:26.216227: 
+epoch:  10 
+2022-04-10 11:57:11.079092: train loss : 0.1531 
+2022-04-10 11:57:37.507640: validation loss: 0.1659 
+2022-04-10 11:57:37.508284: Average global foreground Dice: [0.6197, 0.7407] 
+2022-04-10 11:57:37.508372: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 11:57:37.895661: lr: 0.0003 
+2022-04-10 11:57:37.895844: saving scheduled checkpoint file... 
+2022-04-10 11:57:37.952314: saving checkpoint... 
+2022-04-10 11:57:38.733987: done, saving took 0.84 seconds 
+2022-04-10 11:57:38.746520: done 
+2022-04-10 11:57:38.778751: saving checkpoint... 
+2022-04-10 11:57:39.537382: done, saving took 0.79 seconds 
+2022-04-10 11:58:19.773912: This epoch took 533.557579 s
+ 
+2022-04-10 11:58:19.774240: 
+epoch:  11 
+2022-04-10 12:06:04.175954: train loss : 0.1548 
+2022-04-10 12:06:30.593664: validation loss: 0.1382 
+2022-04-10 12:06:30.594279: Average global foreground Dice: [0.6397, 0.7383] 
+2022-04-10 12:06:30.594364: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 12:06:30.982803: lr: 0.0003 
+2022-04-10 12:06:30.982992: saving scheduled checkpoint file... 
+2022-04-10 12:06:31.040812: saving checkpoint... 
+2022-04-10 12:06:31.837369: done, saving took 0.85 seconds 
+2022-04-10 12:06:31.851191: done 
+2022-04-10 12:06:31.885658: saving checkpoint... 
+2022-04-10 12:06:32.658396: done, saving took 0.81 seconds 
+2022-04-10 12:06:50.596610: This epoch took 510.822263 s
+ 
+2022-04-10 12:06:50.596882: 
+epoch:  12 
+2022-04-10 12:14:35.233165: train loss : 0.1539 
+2022-04-10 12:15:01.674145: validation loss: 0.1393 
+2022-04-10 12:15:01.674678: Average global foreground Dice: [0.64, 0.7559] 
+2022-04-10 12:15:01.674765: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 12:15:02.080619: lr: 0.000299 
+2022-04-10 12:15:02.080812: saving scheduled checkpoint file... 
+2022-04-10 12:15:02.136913: saving checkpoint... 
+2022-04-10 12:15:03.049953: done, saving took 0.97 seconds 
+2022-04-10 12:15:03.069047: done 
+2022-04-10 12:15:03.110219: saving checkpoint... 
+2022-04-10 12:15:03.924820: done, saving took 0.86 seconds 
+2022-04-10 12:15:23.449942: This epoch took 512.852957 s
+ 
+2022-04-10 12:15:23.450258: 
+epoch:  13 
+2022-04-10 12:23:07.565952: train loss : 0.1443 
+2022-04-10 12:23:33.993439: validation loss: 0.1450 
+2022-04-10 12:23:33.994024: Average global foreground Dice: [0.6511, 0.7246] 
+2022-04-10 12:23:33.994111: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 12:23:34.394005: lr: 0.000299 
+2022-04-10 12:23:34.394190: saving scheduled checkpoint file... 
+2022-04-10 12:23:34.449370: saving checkpoint... 
+2022-04-10 12:23:35.260366: done, saving took 0.87 seconds 
+2022-04-10 12:23:35.274855: done 
+2022-04-10 12:23:35.308610: saving checkpoint... 
+2022-04-10 12:23:36.161995: done, saving took 0.89 seconds 
+2022-04-10 12:24:00.189371: This epoch took 516.739017 s
+ 
+2022-04-10 12:24:00.189641: 
+epoch:  14 
+2022-04-10 12:31:44.896982: train loss : 0.1400 
+2022-04-10 12:32:11.312597: validation loss: 0.1379 
+2022-04-10 12:32:11.313218: Average global foreground Dice: [0.6537, 0.7555] 
+2022-04-10 12:32:11.313303: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 12:32:11.705767: lr: 0.000299 
+2022-04-10 12:32:11.705957: saving scheduled checkpoint file... 
+2022-04-10 12:32:11.762720: saving checkpoint... 
+2022-04-10 12:32:12.750886: done, saving took 1.04 seconds 
+2022-04-10 12:32:12.764289: done 
+2022-04-10 12:32:12.797271: saving checkpoint... 
+2022-04-10 12:32:13.818667: done, saving took 1.05 seconds 
+2022-04-10 12:32:30.088279: This epoch took 509.898535 s
+ 
+2022-04-10 12:32:30.088550: 
+epoch:  15 
+2022-04-10 12:40:13.953102: train loss : 0.1418 
+2022-04-10 12:40:40.372629: validation loss: 0.1334 
+2022-04-10 12:40:40.373260: Average global foreground Dice: [0.6209, 0.7783] 
+2022-04-10 12:40:40.373347: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 12:40:40.763979: lr: 0.000298 
+2022-04-10 12:40:40.764187: saving scheduled checkpoint file... 
+2022-04-10 12:40:40.819943: saving checkpoint... 
+2022-04-10 12:40:41.723094: done, saving took 0.96 seconds 
+2022-04-10 12:40:41.736442: done 
+2022-04-10 12:40:41.765638: saving checkpoint... 
+2022-04-10 12:40:42.644211: done, saving took 0.91 seconds 
+2022-04-10 12:41:06.292978: This epoch took 516.204326 s
+ 
+2022-04-10 12:41:06.293259: 
+epoch:  16 
+2022-04-10 12:48:50.477393: train loss : 0.1384 
+2022-04-10 12:49:16.901909: validation loss: 0.1506 
+2022-04-10 12:49:16.902626: Average global foreground Dice: [0.6653, 0.7789] 
+2022-04-10 12:49:16.902716: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 12:49:17.309275: lr: 0.000297 
+2022-04-10 12:49:17.309469: saving scheduled checkpoint file... 
+2022-04-10 12:49:17.366681: saving checkpoint... 
+2022-04-10 12:49:18.191713: done, saving took 0.88 seconds 
+2022-04-10 12:49:18.204602: done 
+2022-04-10 12:49:18.236095: saving checkpoint... 
+2022-04-10 12:49:19.085500: done, saving took 0.88 seconds 
+2022-04-10 12:49:46.800288: This epoch took 520.506924 s
+ 
+2022-04-10 12:49:46.800585: 
+epoch:  17 
+2022-04-10 12:57:31.090198: train loss : 0.1419 
+2022-04-10 12:57:57.504947: validation loss: 0.1332 
+2022-04-10 12:57:57.505548: Average global foreground Dice: [0.6531, 0.7953] 
+2022-04-10 12:57:57.505652: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 12:57:57.899378: lr: 0.000296 
+2022-04-10 12:57:57.899566: saving scheduled checkpoint file... 
+2022-04-10 12:57:57.955918: saving checkpoint... 
+2022-04-10 12:57:58.984966: done, saving took 1.09 seconds 
+2022-04-10 12:57:58.997235: done 
+2022-04-10 12:57:59.026696: saving checkpoint... 
+2022-04-10 12:58:00.024369: done, saving took 1.03 seconds 
+2022-04-10 12:58:19.962373: This epoch took 513.161687 s
+ 
+2022-04-10 12:58:19.962636: 
+epoch:  18 
+2022-04-10 13:06:04.291584: train loss : 0.1330 
+2022-04-10 13:06:30.691805: validation loss: 0.1220 
+2022-04-10 13:06:30.692442: Average global foreground Dice: [0.6467, 0.7856] 
+2022-04-10 13:06:30.692527: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 13:06:31.100821: lr: 0.000295 
+2022-04-10 13:06:31.101003: saving scheduled checkpoint file... 
+2022-04-10 13:06:31.156988: saving checkpoint... 
+2022-04-10 13:06:31.974964: done, saving took 0.87 seconds 
+2022-04-10 13:06:31.989408: done 
+2022-04-10 13:06:32.027232: saving checkpoint... 
+2022-04-10 13:06:32.907462: done, saving took 0.92 seconds 
+2022-04-10 13:06:52.191609: This epoch took 512.228871 s
+ 
+2022-04-10 13:06:52.191895: 
+epoch:  19 
+2022-04-10 13:14:36.116318: train loss : 0.1297 
+2022-04-10 13:15:02.543769: validation loss: 0.1172 
+2022-04-10 13:15:02.544345: Average global foreground Dice: [0.6459, 0.7657] 
+2022-04-10 13:15:02.544434: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 13:15:02.940616: lr: 0.000294 
+2022-04-10 13:15:02.940822: saving scheduled checkpoint file... 
+2022-04-10 13:15:02.997762: saving checkpoint... 
+2022-04-10 13:15:03.878990: done, saving took 0.94 seconds 
+2022-04-10 13:15:03.892158: done 
+2022-04-10 13:15:03.920777: saving checkpoint... 
+2022-04-10 13:15:04.813559: done, saving took 0.92 seconds 
+2022-04-10 13:15:22.673501: This epoch took 510.481505 s
+ 
+2022-04-10 13:15:22.673996: 
+epoch:  20 
+2022-04-10 13:23:06.657405: train loss : 0.1352 
+2022-04-10 13:23:33.083849: validation loss: 0.1349 
+2022-04-10 13:23:33.084473: Average global foreground Dice: [0.6714, 0.808] 
+2022-04-10 13:23:33.084561: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 13:23:33.480651: lr: 0.000293 
+2022-04-10 13:23:33.480853: saving scheduled checkpoint file... 
+2022-04-10 13:23:33.538591: saving checkpoint... 
+2022-04-10 13:23:34.356267: done, saving took 0.88 seconds 
+2022-04-10 13:23:34.369095: done 
+2022-04-10 13:23:34.399544: saving checkpoint... 
+2022-04-10 13:23:35.427487: done, saving took 1.06 seconds 
+2022-04-10 13:23:53.897202: This epoch took 511.223103 s
+ 
+2022-04-10 13:23:53.897502: 
+epoch:  21 
+2022-04-10 13:31:38.240856: train loss : 0.1283 
+2022-04-10 13:32:04.720810: validation loss: 0.1248 
+2022-04-10 13:32:04.721455: Average global foreground Dice: [0.6588, 0.7597] 
+2022-04-10 13:32:04.721545: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 13:32:05.110909: lr: 0.000292 
+2022-04-10 13:32:05.111096: saving scheduled checkpoint file... 
+2022-04-10 13:32:05.166841: saving checkpoint... 
+2022-04-10 13:32:06.163257: done, saving took 1.05 seconds 
+2022-04-10 13:32:06.176237: done 
+2022-04-10 13:32:06.206009: saving checkpoint... 
+2022-04-10 13:32:07.227566: done, saving took 1.05 seconds 
+2022-04-10 13:32:28.861050: This epoch took 514.963443 s
+ 
+2022-04-10 13:32:28.861373: 
+epoch:  22 
+2022-04-10 13:40:13.694591: train loss : 0.1362 
+2022-04-10 13:40:40.203345: validation loss: 0.1339 
+2022-04-10 13:40:40.203902: Average global foreground Dice: [0.6557, 0.7788] 
+2022-04-10 13:40:40.203988: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 13:40:40.595093: lr: 0.000291 
+2022-04-10 13:40:40.595306: saving scheduled checkpoint file... 
+2022-04-10 13:40:40.652963: saving checkpoint... 
+2022-04-10 13:40:41.539101: done, saving took 0.94 seconds 
+2022-04-10 13:40:41.552782: done 
+2022-04-10 13:40:41.581458: saving checkpoint... 
+2022-04-10 13:40:42.494650: done, saving took 0.94 seconds 
+2022-04-10 13:41:04.680154: This epoch took 515.818670 s
+ 
+2022-04-10 13:41:04.680424: 
+epoch:  23 
+2022-04-10 13:48:49.546362: train loss : 0.1318 
+2022-04-10 13:49:15.969859: validation loss: 0.1363 
+2022-04-10 13:49:15.970432: Average global foreground Dice: [0.6691, 0.8296] 
+2022-04-10 13:49:15.970520: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 13:49:16.378339: lr: 0.000289 
+2022-04-10 13:49:16.378529: saving scheduled checkpoint file... 
+2022-04-10 13:49:16.435903: saving checkpoint... 
+2022-04-10 13:49:17.250283: done, saving took 0.87 seconds 
+2022-04-10 13:49:17.263196: done 
+2022-04-10 13:49:17.292560: saving checkpoint... 
+2022-04-10 13:49:18.140007: done, saving took 0.88 seconds 
+2022-04-10 13:49:51.972338: This epoch took 527.291800 s
+ 
+2022-04-10 13:49:51.972694: 
+epoch:  24 
+2022-04-10 13:57:35.897557: train loss : 0.1264 
+2022-04-10 13:58:02.365453: validation loss: 0.1209 
+2022-04-10 13:58:02.366063: Average global foreground Dice: [0.673, 0.8007] 
+2022-04-10 13:58:02.366160: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 13:58:02.757211: lr: 0.000288 
+2022-04-10 13:58:02.757390: saving scheduled checkpoint file... 
+2022-04-10 13:58:02.814049: saving checkpoint... 
+2022-04-10 13:58:03.634842: done, saving took 0.88 seconds 
+2022-04-10 13:58:03.647964: done 
+2022-04-10 13:58:03.677928: saving checkpoint... 
+2022-04-10 13:58:04.668735: done, saving took 1.02 seconds 
+2022-04-10 13:58:25.510682: This epoch took 513.537880 s
+ 
+2022-04-10 13:58:25.510972: 
+epoch:  25 
+2022-04-10 14:06:10.291184: train loss : 0.1344 
+2022-04-10 14:06:36.814219: validation loss: 0.1457 
+2022-04-10 14:06:36.814806: Average global foreground Dice: [0.6567, 0.7982] 
+2022-04-10 14:06:36.814891: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 14:06:37.206000: lr: 0.000286 
+2022-04-10 14:06:37.206203: saving scheduled checkpoint file... 
+2022-04-10 14:06:37.263176: saving checkpoint... 
+2022-04-10 14:06:38.077608: done, saving took 0.87 seconds 
+2022-04-10 14:06:38.092010: done 
+2022-04-10 14:06:38.125357: saving checkpoint... 
+2022-04-10 14:06:38.978630: done, saving took 0.89 seconds 
+2022-04-10 14:07:20.580197: This epoch took 535.069123 s
+ 
+2022-04-10 14:07:20.580524: 
+epoch:  26 
+2022-04-10 14:15:04.843510: train loss : 0.1285 
+2022-04-10 14:15:31.288652: validation loss: 0.1136 
+2022-04-10 14:15:31.289351: Average global foreground Dice: [0.6717, 0.8117] 
+2022-04-10 14:15:31.289440: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 14:15:31.681301: lr: 0.000284 
+2022-04-10 14:15:31.681486: saving scheduled checkpoint file... 
+2022-04-10 14:15:31.737681: saving checkpoint... 
+2022-04-10 14:15:32.560060: done, saving took 0.88 seconds 
+2022-04-10 14:15:32.574474: done 
+2022-04-10 14:15:32.607931: saving checkpoint... 
+2022-04-10 14:15:33.546102: done, saving took 0.97 seconds 
+2022-04-10 14:15:51.125623: This epoch took 510.544989 s
+ 
+2022-04-10 14:15:51.125941: 
+epoch:  27 
+2022-04-10 14:23:35.668289: train loss : 0.1283 
+2022-04-10 14:24:02.100888: validation loss: 0.1354 
+2022-04-10 14:24:02.101506: Average global foreground Dice: [0.6746, 0.8268] 
+2022-04-10 14:24:02.101593: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 14:24:02.501421: lr: 0.000282 
+2022-04-10 14:24:02.501620: saving scheduled checkpoint file... 
+2022-04-10 14:24:02.558971: saving checkpoint... 
+2022-04-10 14:24:03.374263: done, saving took 0.87 seconds 
+2022-04-10 14:24:03.387265: done 
+2022-04-10 14:24:03.417263: saving checkpoint... 
+2022-04-10 14:24:04.248475: done, saving took 0.86 seconds 
+2022-04-10 14:24:22.496168: This epoch took 511.370062 s
+ 
+2022-04-10 14:24:22.496541: 
+epoch:  28 
+2022-04-10 14:32:06.920557: train loss : 0.1259 
+2022-04-10 14:32:33.388004: validation loss: 0.1439 
+2022-04-10 14:32:33.388676: Average global foreground Dice: [0.6538, 0.7741] 
+2022-04-10 14:32:33.388770: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 14:32:33.793237: lr: 0.00028 
+2022-04-10 14:32:33.793420: saving scheduled checkpoint file... 
+2022-04-10 14:32:33.849550: saving checkpoint... 
+2022-04-10 14:32:34.879028: done, saving took 1.09 seconds 
+2022-04-10 14:32:34.892491: done 
+2022-04-10 14:32:34.922424: saving checkpoint... 
+2022-04-10 14:32:35.899621: done, saving took 1.01 seconds 
+2022-04-10 14:33:03.751230: This epoch took 521.254506 s
+ 
+2022-04-10 14:33:03.751529: 
+epoch:  29 
+2022-04-10 14:40:47.992644: train loss : 0.1316 
+2022-04-10 14:41:14.459565: validation loss: 0.1396 
+2022-04-10 14:41:14.460195: Average global foreground Dice: [0.6596, 0.813] 
+2022-04-10 14:41:14.460285: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 14:41:14.857873: lr: 0.000278 
+2022-04-10 14:41:14.858066: saving scheduled checkpoint file... 
+2022-04-10 14:41:14.916051: saving checkpoint... 
+2022-04-10 14:41:15.740700: done, saving took 0.88 seconds 
+2022-04-10 14:41:15.753756: done 
+2022-04-10 14:41:15.784206: saving checkpoint... 
+2022-04-10 14:41:16.630979: done, saving took 0.88 seconds 
+2022-04-10 14:41:40.002280: This epoch took 516.250626 s
+ 
+2022-04-10 14:41:40.002810: 
+epoch:  30 
+2022-04-10 14:49:25.384807: train loss : 0.1219 
+2022-04-10 14:49:51.780580: validation loss: 0.1328 
+2022-04-10 14:49:51.781229: Average global foreground Dice: [0.6734, 0.8252] 
+2022-04-10 14:49:51.781315: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 14:49:52.174561: lr: 0.000276 
+2022-04-10 14:49:52.174739: saving scheduled checkpoint file... 
+2022-04-10 14:49:52.230343: saving checkpoint... 
+2022-04-10 14:49:53.044765: done, saving took 0.87 seconds 
+2022-04-10 14:49:53.057697: done 
+2022-04-10 14:49:53.087986: saving checkpoint... 
+2022-04-10 14:49:53.872538: done, saving took 0.81 seconds 
+2022-04-10 14:50:14.876592: This epoch took 514.873681 s
+ 
+2022-04-10 14:50:14.876891: 
+epoch:  31 
+2022-04-10 14:57:59.018251: train loss : 0.1221 
+2022-04-10 14:58:25.482395: validation loss: 0.1302 
+2022-04-10 14:58:25.483011: Average global foreground Dice: [0.6671, 0.8113] 
+2022-04-10 14:58:25.483101: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 14:58:25.963676: lr: 0.000274 
+2022-04-10 14:58:25.963878: saving scheduled checkpoint file... 
+2022-04-10 14:58:26.020991: saving checkpoint... 
+2022-04-10 14:58:26.843817: done, saving took 0.88 seconds 
+2022-04-10 14:58:26.857701: done 
+2022-04-10 14:58:26.887722: saving checkpoint... 
+2022-04-10 14:58:27.723910: done, saving took 0.87 seconds 
+2022-04-10 14:58:46.158249: This epoch took 511.281236 s
+ 
+2022-04-10 14:58:46.158544: 
+epoch:  32 
+2022-04-10 15:06:30.691526: train loss : 0.1242 
+2022-04-10 15:06:57.198744: validation loss: 0.1377 
+2022-04-10 15:06:57.199334: Average global foreground Dice: [0.6668, 0.78] 
+2022-04-10 15:06:57.199420: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 15:06:57.592738: lr: 0.000271 
+2022-04-10 15:06:57.592927: saving scheduled checkpoint file... 
+2022-04-10 15:06:57.649013: saving checkpoint... 
+2022-04-10 15:06:58.489680: done, saving took 0.90 seconds 
+2022-04-10 15:06:58.502633: done 
+2022-04-10 15:06:58.531625: saving checkpoint... 
+2022-04-10 15:06:59.443648: done, saving took 0.94 seconds 
+2022-04-10 15:07:32.971561: This epoch took 526.812918 s
+ 
+2022-04-10 15:07:32.971838: 
+epoch:  33 
+2022-04-10 15:15:16.335568: train loss : 0.1276 
+2022-04-10 15:15:42.779854: validation loss: 0.1262 
+2022-04-10 15:15:42.780493: Average global foreground Dice: [0.6582, 0.7951] 
+2022-04-10 15:15:42.780579: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 15:15:43.174975: lr: 0.000269 
+2022-04-10 15:15:43.175161: saving scheduled checkpoint file... 
+2022-04-10 15:15:43.233563: saving checkpoint... 
+2022-04-10 15:15:44.046446: done, saving took 0.87 seconds 
+2022-04-10 15:15:44.061023: done 
+2022-04-10 15:15:44.093986: saving checkpoint... 
+2022-04-10 15:15:44.937058: done, saving took 0.88 seconds 
+2022-04-10 15:16:09.125626: This epoch took 516.153557 s
+ 
+2022-04-10 15:16:09.125950: 
+epoch:  34 
+2022-04-10 15:23:53.500041: train loss : 0.1234 
+2022-04-10 15:24:19.916746: validation loss: 0.1222 
+2022-04-10 15:24:19.917358: Average global foreground Dice: [0.6833, 0.8191] 
+2022-04-10 15:24:19.917466: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 15:24:20.319601: lr: 0.000266 
+2022-04-10 15:24:20.319795: saving scheduled checkpoint file... 
+2022-04-10 15:24:20.377032: saving checkpoint... 
+2022-04-10 15:24:21.205632: done, saving took 0.89 seconds 
+2022-04-10 15:24:21.218979: done 
+2022-04-10 15:24:21.249791: saving checkpoint... 
+2022-04-10 15:24:22.103273: done, saving took 0.88 seconds 
+2022-04-10 15:25:04.843268: This epoch took 535.717218 s
+ 
+2022-04-10 15:25:04.843692: 
+epoch:  35 
+2022-04-10 15:32:49.050870: train loss : 0.1262 
+2022-04-10 15:33:15.471733: validation loss: 0.1308 
+2022-04-10 15:33:15.472346: Average global foreground Dice: [0.6841, 0.8307] 
+2022-04-10 15:33:15.472435: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 15:33:15.868579: lr: 0.000264 
+2022-04-10 15:33:15.868767: saving scheduled checkpoint file... 
+2022-04-10 15:33:15.926542: saving checkpoint... 
+2022-04-10 15:33:16.742320: done, saving took 0.87 seconds 
+2022-04-10 15:33:16.755079: done 
+2022-04-10 15:33:16.785022: saving checkpoint... 
+2022-04-10 15:33:17.623916: done, saving took 0.87 seconds 
+2022-04-10 15:34:44.608119: This epoch took 579.764162 s
+ 
+2022-04-10 15:34:44.608399: 
+epoch:  36 
+2022-04-10 15:42:28.694982: train loss : 0.1238 
+2022-04-10 15:42:55.139494: validation loss: 0.1320 
+2022-04-10 15:42:55.140136: Average global foreground Dice: [0.6764, 0.8164] 
+2022-04-10 15:42:55.140226: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 15:42:55.536073: lr: 0.000261 
+2022-04-10 15:42:55.536278: saving scheduled checkpoint file... 
+2022-04-10 15:42:55.593449: saving checkpoint... 
+2022-04-10 15:42:56.481532: done, saving took 0.95 seconds 
+2022-04-10 15:42:56.494972: done 
+2022-04-10 15:42:56.524474: saving checkpoint... 
+2022-04-10 15:42:57.390097: done, saving took 0.89 seconds 
+2022-04-10 15:43:15.633323: This epoch took 511.024823 s
+ 
+2022-04-10 15:43:15.633590: 
+epoch:  37 
+2022-04-10 15:50:59.355224: train loss : 0.1200 
+2022-04-10 15:51:25.819361: validation loss: 0.1254 
+2022-04-10 15:51:25.819967: Average global foreground Dice: [0.6697, 0.8039] 
+2022-04-10 15:51:25.820054: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 15:51:26.215863: lr: 0.000258 
+2022-04-10 15:51:26.216044: saving scheduled checkpoint file... 
+2022-04-10 15:51:26.272840: saving checkpoint... 
+2022-04-10 15:51:27.092546: done, saving took 0.88 seconds 
+2022-04-10 15:51:27.105568: done 
+2022-04-10 15:51:27.135628: saving checkpoint... 
+2022-04-10 15:51:27.958562: done, saving took 0.85 seconds 
+2022-04-10 15:51:48.602489: This epoch took 512.968800 s
+ 
+2022-04-10 15:51:48.602772: 
+epoch:  38 
+2022-04-10 15:59:33.006900: train loss : 0.1190 
+2022-04-10 15:59:59.450432: validation loss: 0.1380 
+2022-04-10 15:59:59.451063: Average global foreground Dice: [0.6761, 0.8213] 
+2022-04-10 15:59:59.451165: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 15:59:59.850525: lr: 0.000255 
+2022-04-10 15:59:59.850713: saving scheduled checkpoint file... 
+2022-04-10 15:59:59.907736: saving checkpoint... 
+2022-04-10 16:00:00.731169: done, saving took 0.88 seconds 
+2022-04-10 16:00:00.745807: done 
+2022-04-10 16:00:00.779578: saving checkpoint... 
+2022-04-10 16:00:01.617216: done, saving took 0.87 seconds 
+2022-04-10 16:00:33.311090: This epoch took 524.708216 s
+ 
+2022-04-10 16:00:33.311389: 
+epoch:  39 
+2022-04-10 16:08:17.854805: train loss : 0.1224 
+2022-04-10 16:08:44.288782: validation loss: 0.1213 
+2022-04-10 16:08:44.289325: Average global foreground Dice: [0.6668, 0.8076] 
+2022-04-10 16:08:44.289408: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 16:08:44.684845: lr: 0.000252 
+2022-04-10 16:08:44.685032: saving scheduled checkpoint file... 
+2022-04-10 16:08:44.741791: saving checkpoint... 
+2022-04-10 16:08:45.625904: done, saving took 0.94 seconds 
+2022-04-10 16:08:45.638534: done 
+2022-04-10 16:08:45.666790: saving checkpoint... 
+2022-04-10 16:08:46.570761: done, saving took 0.93 seconds 
+2022-04-10 16:09:11.486919: This epoch took 518.175428 s
+ 
+2022-04-10 16:09:11.487210: 
+epoch:  40 
+2022-04-10 16:16:56.085935: train loss : 0.1210 
+2022-04-10 16:17:22.497313: validation loss: 0.1290 
+2022-04-10 16:17:22.498268: Average global foreground Dice: [0.6823, 0.8258] 
+2022-04-10 16:17:22.498356: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 16:17:22.898221: lr: 0.000249 
+2022-04-10 16:17:22.898414: saving scheduled checkpoint file... 
+2022-04-10 16:17:22.955707: saving checkpoint... 
+2022-04-10 16:17:23.826893: done, saving took 0.93 seconds 
+2022-04-10 16:17:23.844111: done 
+2022-04-10 16:17:23.884347: saving checkpoint... 
+2022-04-10 16:17:24.741935: done, saving took 0.90 seconds 
+2022-04-10 16:17:41.128432: This epoch took 509.641095 s
+ 
+2022-04-10 16:17:41.128837: 
+epoch:  41 
+2022-04-10 16:25:25.168788: train loss : 0.1161 
+2022-04-10 16:25:51.590281: validation loss: 0.1345 
+2022-04-10 16:25:51.590930: Average global foreground Dice: [0.6703, 0.8112] 
+2022-04-10 16:25:51.591014: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 16:25:51.982786: lr: 0.000246 
+2022-04-10 16:25:51.982971: saving scheduled checkpoint file... 
+2022-04-10 16:25:52.041010: saving checkpoint... 
+2022-04-10 16:25:52.862645: done, saving took 0.88 seconds 
+2022-04-10 16:25:52.877097: done 
+2022-04-10 16:25:52.910192: saving checkpoint... 
+2022-04-10 16:25:53.717356: done, saving took 0.84 seconds 
+2022-04-10 16:26:10.727857: This epoch took 509.598911 s
+ 
+2022-04-10 16:26:10.728193: 
+epoch:  42 
+2022-04-10 16:33:54.883646: train loss : 0.1175 
+2022-04-10 16:34:21.286144: validation loss: 0.1158 
+2022-04-10 16:34:21.286749: Average global foreground Dice: [0.6801, 0.8278] 
+2022-04-10 16:34:21.286836: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 16:34:21.682561: lr: 0.000243 
+2022-04-10 16:34:21.682735: saving scheduled checkpoint file... 
+2022-04-10 16:34:21.738679: saving checkpoint... 
+2022-04-10 16:34:22.561829: done, saving took 0.88 seconds 
+2022-04-10 16:34:22.576387: done 
+2022-04-10 16:34:22.610392: saving checkpoint... 
+2022-04-10 16:34:23.640694: done, saving took 1.06 seconds 
+2022-04-10 16:34:40.223772: This epoch took 509.495473 s
+ 
+2022-04-10 16:34:40.224072: 
+epoch:  43 
+2022-04-10 16:42:25.034397: train loss : 0.1192 
+2022-04-10 16:42:51.475712: validation loss: 0.1188 
+2022-04-10 16:42:51.476380: Average global foreground Dice: [0.6669, 0.8181] 
+2022-04-10 16:42:51.476470: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 16:42:51.873569: lr: 0.00024 
+2022-04-10 16:42:51.873765: saving scheduled checkpoint file... 
+2022-04-10 16:42:51.932375: saving checkpoint... 
+2022-04-10 16:42:52.872739: done, saving took 1.00 seconds 
+2022-04-10 16:42:52.884951: done 
+2022-04-10 16:42:52.923041: saving checkpoint... 
+2022-04-10 16:42:53.797025: done, saving took 0.91 seconds 
+2022-04-10 16:43:11.915216: This epoch took 511.691028 s
+ 
+2022-04-10 16:43:11.915486: 
+epoch:  44 
+2022-04-10 16:50:56.180829: train loss : 0.1151 
+2022-04-10 16:51:22.595149: validation loss: 0.1148 
+2022-04-10 16:51:22.595716: Average global foreground Dice: [0.6638, 0.8237] 
+2022-04-10 16:51:22.595804: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 16:51:22.994210: lr: 0.000237 
+2022-04-10 16:51:22.994389: saving scheduled checkpoint file... 
+2022-04-10 16:51:23.051076: saving checkpoint... 
+2022-04-10 16:51:23.874643: done, saving took 0.88 seconds 
+2022-04-10 16:51:23.887641: done 
+2022-04-10 16:51:23.917653: saving checkpoint... 
+2022-04-10 16:51:24.744459: done, saving took 0.86 seconds 
+2022-04-10 16:51:48.896856: This epoch took 516.981267 s
+ 
+2022-04-10 16:51:48.897141: 
+epoch:  45 
+2022-04-10 16:59:33.551281: train loss : 0.1183 
+2022-04-10 16:59:59.957427: validation loss: 0.1165 
+2022-04-10 16:59:59.957966: Average global foreground Dice: [0.6728, 0.8305] 
+2022-04-10 16:59:59.958050: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 17:00:00.346981: lr: 0.000233 
+2022-04-10 17:00:00.347173: saving scheduled checkpoint file... 
+2022-04-10 17:00:00.403191: saving checkpoint... 
+2022-04-10 17:00:01.224226: done, saving took 0.88 seconds 
+2022-04-10 17:00:01.237031: done 
+2022-04-10 17:00:01.266955: saving checkpoint... 
+2022-04-10 17:00:02.090358: done, saving took 0.85 seconds 
+2022-04-10 17:00:21.914198: This epoch took 513.016935 s
+ 
+2022-04-10 17:00:21.914511: 
+epoch:  46 
+2022-04-10 17:08:06.070720: train loss : 0.1206 
+2022-04-10 17:08:32.516577: validation loss: 0.1104 
+2022-04-10 17:08:32.517176: Average global foreground Dice: [0.6927, 0.833] 
+2022-04-10 17:08:32.517264: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 17:08:32.908116: lr: 0.00023 
+2022-04-10 17:08:32.908305: saving scheduled checkpoint file... 
+2022-04-10 17:08:32.965157: saving checkpoint... 
+2022-04-10 17:08:33.775437: done, saving took 0.87 seconds 
+2022-04-10 17:08:33.789555: done 
+2022-04-10 17:08:33.823405: saving checkpoint... 
+2022-04-10 17:08:34.687897: done, saving took 0.90 seconds 
+2022-04-10 17:08:52.124776: This epoch took 510.210159 s
+ 
+2022-04-10 17:08:52.125062: 
+epoch:  47 
+2022-04-10 17:16:36.440623: train loss : 0.1122 
+2022-04-10 17:17:02.865947: validation loss: 0.1064 
+2022-04-10 17:17:02.866593: Average global foreground Dice: [0.6858, 0.8226] 
+2022-04-10 17:17:02.866681: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 17:17:03.261701: lr: 0.000226 
+2022-04-10 17:17:03.261898: saving scheduled checkpoint file... 
+2022-04-10 17:17:03.318717: saving checkpoint... 
+2022-04-10 17:17:04.221603: done, saving took 0.96 seconds 
+2022-04-10 17:17:04.235050: done 
+2022-04-10 17:17:04.264794: saving checkpoint... 
+2022-04-10 17:17:05.165858: done, saving took 0.93 seconds 
+2022-04-10 17:17:20.870166: This epoch took 508.744987 s
+ 
+2022-04-10 17:17:20.870448: 
+epoch:  48 
+2022-04-10 17:25:05.172134: train loss : 0.1208 
+2022-04-10 17:25:31.626356: validation loss: 0.1248 
+2022-04-10 17:25:31.627004: Average global foreground Dice: [0.67, 0.8427] 
+2022-04-10 17:25:31.627095: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 17:25:32.025546: lr: 0.000223 
+2022-04-10 17:25:32.025745: saving scheduled checkpoint file... 
+2022-04-10 17:25:32.081989: saving checkpoint... 
+2022-04-10 17:25:32.911008: done, saving took 0.89 seconds 
+2022-04-10 17:25:32.924431: done 
+2022-04-10 17:25:32.954215: saving checkpoint... 
+2022-04-10 17:25:33.777052: done, saving took 0.85 seconds 
+2022-04-10 17:25:50.387898: This epoch took 509.517355 s
+ 
+2022-04-10 17:25:50.388192: 
+epoch:  49 
+2022-04-10 17:33:34.298388: train loss : 0.1160 
+2022-04-10 17:34:00.759029: validation loss: 0.1186 
+2022-04-10 17:34:00.759623: Average global foreground Dice: [0.6781, 0.8164] 
+2022-04-10 17:34:00.759709: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 17:34:01.154538: lr: 0.000219 
+2022-04-10 17:34:01.154736: saving scheduled checkpoint file... 
+2022-04-10 17:34:01.212195: saving checkpoint... 
+2022-04-10 17:34:02.036551: done, saving took 0.88 seconds 
+2022-04-10 17:34:02.049695: done 
+2022-04-10 17:34:02.081493: saving checkpoint... 
+2022-04-10 17:34:03.100510: done, saving took 1.05 seconds 
+2022-04-10 17:34:20.323286: This epoch took 509.934981 s
+ 
+2022-04-10 17:34:20.323631: 
+epoch:  50 
+2022-04-10 17:42:04.581979: train loss : 0.1102 
+2022-04-10 17:42:31.013095: validation loss: 0.1094 
+2022-04-10 17:42:31.013748: Average global foreground Dice: [0.6877, 0.8159] 
+2022-04-10 17:42:31.013835: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 17:42:31.411103: lr: 0.000215 
+2022-04-10 17:42:31.411344: saving scheduled checkpoint file... 
+2022-04-10 17:42:31.468648: saving checkpoint... 
+2022-04-10 17:42:32.290036: done, saving took 0.88 seconds 
+2022-04-10 17:42:32.302916: done 
+2022-04-10 17:42:32.332727: saving checkpoint... 
+2022-04-10 17:42:33.158389: done, saving took 0.86 seconds 
+2022-04-10 17:42:50.719433: This epoch took 510.395678 s
+ 
+2022-04-10 17:42:50.719709: 
+epoch:  51 
+2022-04-10 17:50:35.299026: train loss : 0.1140 
+2022-04-10 17:51:01.694472: validation loss: 0.1296 
+2022-04-10 17:51:01.695050: Average global foreground Dice: [0.6814, 0.8152] 
+2022-04-10 17:51:01.695205: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 17:51:02.091885: lr: 0.000212 
+2022-04-10 17:51:02.092079: saving scheduled checkpoint file... 
+2022-04-10 17:51:02.148270: saving checkpoint... 
+2022-04-10 17:51:03.026335: done, saving took 0.93 seconds 
+2022-04-10 17:51:03.040011: done 
+2022-04-10 17:51:03.070823: saving checkpoint... 
+2022-04-10 17:51:03.920826: done, saving took 0.88 seconds 
+2022-04-10 17:51:20.021781: This epoch took 509.301962 s
+ 
+2022-04-10 17:51:20.022107: 
+epoch:  52 
+2022-04-10 17:59:03.999905: train loss : 0.1107 
+2022-04-10 17:59:30.398362: validation loss: 0.1222 
+2022-04-10 17:59:30.398919: Average global foreground Dice: [0.6784, 0.8383] 
+2022-04-10 17:59:30.399002: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 17:59:30.794969: lr: 0.000208 
+2022-04-10 17:59:30.795152: saving scheduled checkpoint file... 
+2022-04-10 17:59:30.852326: saving checkpoint... 
+2022-04-10 17:59:31.771925: done, saving took 0.98 seconds 
+2022-04-10 17:59:31.784384: done 
+2022-04-10 17:59:31.813557: saving checkpoint... 
+2022-04-10 17:59:32.740166: done, saving took 0.96 seconds 
+2022-04-10 17:59:59.598258: This epoch took 519.576030 s
+ 
+2022-04-10 17:59:59.598573: 
+epoch:  53 
+2022-04-10 18:07:43.210997: train loss : 0.1139 
+2022-04-10 18:08:09.612471: validation loss: 0.1190 
+2022-04-10 18:08:09.613055: Average global foreground Dice: [0.676, 0.7978] 
+2022-04-10 18:08:09.613152: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 18:08:10.010361: lr: 0.000204 
+2022-04-10 18:08:10.010548: saving scheduled checkpoint file... 
+2022-04-10 18:08:10.067865: saving checkpoint... 
+2022-04-10 18:08:11.062145: done, saving took 1.05 seconds 
+2022-04-10 18:08:11.075934: done 
+2022-04-10 18:08:28.214177: This epoch took 508.615498 s
+ 
+2022-04-10 18:08:28.214434: 
+epoch:  54 
+2022-04-10 18:16:12.332705: train loss : 0.1131 
+2022-04-10 18:16:38.747146: validation loss: 0.1268 
+2022-04-10 18:16:38.747795: Average global foreground Dice: [0.6827, 0.8205] 
+2022-04-10 18:16:38.747883: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 18:16:39.152310: lr: 0.0002 
+2022-04-10 18:16:39.152492: saving scheduled checkpoint file... 
+2022-04-10 18:16:39.209754: saving checkpoint... 
+2022-04-10 18:16:40.235250: done, saving took 1.08 seconds 
+2022-04-10 18:16:40.255110: done 
+2022-04-10 18:17:00.607598: This epoch took 512.393060 s
+ 
+2022-04-10 18:17:00.607884: 
+epoch:  55 
+2022-04-10 18:24:44.721942: train loss : 0.1144 
+2022-04-10 18:25:11.155861: validation loss: 0.1068 
+2022-04-10 18:25:11.156523: Average global foreground Dice: [0.6799, 0.8303] 
+2022-04-10 18:25:11.156612: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 18:25:11.551606: lr: 0.000196 
+2022-04-10 18:25:11.551789: saving scheduled checkpoint file... 
+2022-04-10 18:25:11.609357: saving checkpoint... 
+2022-04-10 18:25:12.632370: done, saving took 1.08 seconds 
+2022-04-10 18:25:12.645816: done 
+2022-04-10 18:25:12.679531: saving checkpoint... 
+2022-04-10 18:25:13.691989: done, saving took 1.05 seconds 
+2022-04-10 18:25:32.792464: This epoch took 512.184472 s
+ 
+2022-04-10 18:25:32.792747: 
+epoch:  56 
+2022-04-10 18:33:17.584821: train loss : 0.1065 
+2022-04-10 18:33:43.997276: validation loss: 0.1181 
+2022-04-10 18:33:43.997907: Average global foreground Dice: [0.678, 0.8407] 
+2022-04-10 18:33:43.997996: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 18:33:44.394066: lr: 0.000192 
+2022-04-10 18:33:44.394270: saving scheduled checkpoint file... 
+2022-04-10 18:33:44.450553: saving checkpoint... 
+2022-04-10 18:33:45.427743: done, saving took 1.03 seconds 
+2022-04-10 18:33:45.441588: done 
+2022-04-10 18:33:45.474707: saving checkpoint... 
+2022-04-10 18:33:46.468298: done, saving took 1.03 seconds 
+2022-04-10 18:34:08.129767: This epoch took 515.336923 s
+ 
+2022-04-10 18:34:08.130035: 
+epoch:  57 
+2022-04-10 18:41:51.971055: train loss : 0.1110 
+2022-04-10 18:42:18.381555: validation loss: 0.1133 
+2022-04-10 18:42:18.382137: Average global foreground Dice: [0.6701, 0.8503] 
+2022-04-10 18:42:18.382223: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 18:42:18.778683: lr: 0.000188 
+2022-04-10 18:42:18.778867: saving scheduled checkpoint file... 
+2022-04-10 18:42:18.835061: saving checkpoint... 
+2022-04-10 18:42:19.856058: done, saving took 1.08 seconds 
+2022-04-10 18:42:19.868814: done 
+2022-04-10 18:42:19.898291: saving checkpoint... 
+2022-04-10 18:42:20.924872: done, saving took 1.06 seconds 
+2022-04-10 18:42:38.315782: This epoch took 510.185647 s
+ 
+2022-04-10 18:42:38.316048: 
+epoch:  58 
+2022-04-10 18:50:22.589246: train loss : 0.1072 
+2022-04-10 18:50:49.023482: validation loss: 0.1275 
+2022-04-10 18:50:49.024100: Average global foreground Dice: [0.6819, 0.8505] 
+2022-04-10 18:50:49.024200: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 18:50:49.425480: lr: 0.000185 
+2022-04-10 18:50:49.425661: saving scheduled checkpoint file... 
+2022-04-10 18:50:49.483827: saving checkpoint... 
+2022-04-10 18:50:50.378861: done, saving took 0.95 seconds 
+2022-04-10 18:50:50.392459: done 
+2022-04-10 18:50:50.421762: saving checkpoint... 
+2022-04-10 18:50:51.184623: done, saving took 0.79 seconds 
+2022-04-10 18:51:08.298443: This epoch took 509.982294 s
+ 
+2022-04-10 18:51:08.298739: 
+epoch:  59 
+2022-04-10 18:58:52.447703: train loss : 0.1149 
+2022-04-10 18:59:18.876377: validation loss: 0.1082 
+2022-04-10 18:59:18.876933: Average global foreground Dice: [0.6657, 0.8336] 
+2022-04-10 18:59:18.877015: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 18:59:19.275070: lr: 0.000181 
+2022-04-10 18:59:19.275262: saving scheduled checkpoint file... 
+2022-04-10 18:59:19.332957: saving checkpoint... 
+2022-04-10 18:59:20.144989: done, saving took 0.87 seconds 
+2022-04-10 18:59:20.157806: done 
+2022-04-10 18:59:39.136008: This epoch took 510.837161 s
+ 
+2022-04-10 18:59:39.136494: 
+epoch:  60 
+2022-04-10 19:07:23.413004: train loss : 0.1055 
+2022-04-10 19:07:49.824942: validation loss: 0.1138 
+2022-04-10 19:07:49.825573: Average global foreground Dice: [0.6866, 0.8298] 
+2022-04-10 19:07:49.825661: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 19:07:50.225639: lr: 0.000176 
+2022-04-10 19:07:50.225836: saving scheduled checkpoint file... 
+2022-04-10 19:07:50.282266: saving checkpoint... 
+2022-04-10 19:07:51.097497: done, saving took 0.87 seconds 
+2022-04-10 19:07:51.110651: done 
+2022-04-10 19:07:51.141016: saving checkpoint... 
+2022-04-10 19:07:51.968139: done, saving took 0.86 seconds 
+2022-04-10 19:08:14.329498: This epoch took 515.192860 s
+ 
+2022-04-10 19:08:14.329780: 
+epoch:  61 
+2022-04-10 19:15:59.130463: train loss : 0.1149 
+2022-04-10 19:16:25.522785: validation loss: 0.1265 
+2022-04-10 19:16:25.523375: Average global foreground Dice: [0.6823, 0.8411] 
+2022-04-10 19:16:25.523461: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 19:16:25.922982: lr: 0.000172 
+2022-04-10 19:16:25.923174: saving scheduled checkpoint file... 
+2022-04-10 19:16:25.979764: saving checkpoint... 
+2022-04-10 19:16:26.806254: done, saving took 0.88 seconds 
+2022-04-10 19:16:26.827599: done 
+2022-04-10 19:16:26.875393: saving checkpoint... 
+2022-04-10 19:16:27.700795: done, saving took 0.87 seconds 
+2022-04-10 19:16:46.722876: This epoch took 512.392988 s
+ 
+2022-04-10 19:16:46.723271: 
+epoch:  62 
+2022-04-10 19:24:30.682276: train loss : 0.1133 
+2022-04-10 19:24:57.073393: validation loss: 0.1242 
+2022-04-10 19:24:57.074032: Average global foreground Dice: [0.6717, 0.819] 
+2022-04-10 19:24:57.074132: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 19:24:57.473785: lr: 0.000168 
+2022-04-10 19:24:57.473975: saving scheduled checkpoint file... 
+2022-04-10 19:24:57.529838: saving checkpoint... 
+2022-04-10 19:24:58.425819: done, saving took 0.95 seconds 
+2022-04-10 19:24:58.439954: done 
+2022-04-10 19:25:16.047675: This epoch took 509.324290 s
+ 
+2022-04-10 19:25:16.047951: 
+epoch:  63 
+2022-04-10 19:33:00.216703: train loss : 0.1116 
+2022-04-10 19:33:26.640915: validation loss: 0.1079 
+2022-04-10 19:33:26.641525: Average global foreground Dice: [0.6792, 0.8322] 
+2022-04-10 19:33:26.641613: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 19:33:27.041514: lr: 0.000164 
+2022-04-10 19:33:27.041698: saving scheduled checkpoint file... 
+2022-04-10 19:33:27.100014: saving checkpoint... 
+2022-04-10 19:33:27.916322: done, saving took 0.87 seconds 
+2022-04-10 19:33:27.930566: done 
+2022-04-10 19:34:00.179866: This epoch took 524.131808 s
+ 
+2022-04-10 19:34:00.180168: 
+epoch:  64 
+2022-04-10 19:41:44.503989: train loss : 0.1053 
+2022-04-10 19:42:10.946545: validation loss: 0.1388 
+2022-04-10 19:42:10.947120: Average global foreground Dice: [0.6806, 0.8359] 
+2022-04-10 19:42:10.947219: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 19:42:11.349726: lr: 0.00016 
+2022-04-10 19:42:11.349920: saving scheduled checkpoint file... 
+2022-04-10 19:42:11.406990: saving checkpoint... 
+2022-04-10 19:42:12.227030: done, saving took 0.88 seconds 
+2022-04-10 19:42:12.241160: done 
+2022-04-10 19:42:12.273988: saving checkpoint... 
+2022-04-10 19:42:13.116432: done, saving took 0.88 seconds 
+2022-04-10 19:42:33.935011: This epoch took 513.754740 s
+ 
+2022-04-10 19:42:33.935300: 
+epoch:  65 
+2022-04-10 19:50:18.183059: train loss : 0.1043 
+2022-04-10 19:50:44.627819: validation loss: 0.1199 
+2022-04-10 19:50:44.628426: Average global foreground Dice: [0.6882, 0.8332] 
+2022-04-10 19:50:44.628519: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 19:50:45.028141: lr: 0.000156 
+2022-04-10 19:50:45.028336: saving scheduled checkpoint file... 
+2022-04-10 19:50:45.084832: saving checkpoint... 
+2022-04-10 19:50:45.992005: done, saving took 0.96 seconds 
+2022-04-10 19:50:46.004952: done 
+2022-04-10 19:50:46.033453: saving checkpoint... 
+2022-04-10 19:50:46.886314: done, saving took 0.88 seconds 
+2022-04-10 19:51:07.609160: This epoch took 513.673750 s
+ 
+2022-04-10 19:51:07.609432: 
+epoch:  66 
+2022-04-10 19:58:52.244639: train loss : 0.1063 
+2022-04-10 19:59:18.726210: validation loss: 0.1061 
+2022-04-10 19:59:18.726827: Average global foreground Dice: [0.6785, 0.8475] 
+2022-04-10 19:59:18.726912: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 19:59:19.126150: lr: 0.000152 
+2022-04-10 19:59:19.126345: saving scheduled checkpoint file... 
+2022-04-10 19:59:19.183133: saving checkpoint... 
+2022-04-10 19:59:20.021074: done, saving took 0.89 seconds 
+2022-04-10 19:59:20.039160: done 
+2022-04-10 19:59:20.081260: saving checkpoint... 
+2022-04-10 19:59:21.088437: done, saving took 1.05 seconds 
+2022-04-10 19:59:46.658662: This epoch took 519.049109 s
+ 
+2022-04-10 19:59:46.659066: 
+epoch:  67 
+2022-04-10 20:07:32.121053: train loss : 0.1109 
+2022-04-10 20:07:58.613213: validation loss: 0.1249 
+2022-04-10 20:07:58.613809: Average global foreground Dice: [0.6774, 0.8245] 
+2022-04-10 20:07:58.613895: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 20:07:59.015267: lr: 0.000148 
+2022-04-10 20:07:59.015451: saving scheduled checkpoint file... 
+2022-04-10 20:07:59.071609: saving checkpoint... 
+2022-04-10 20:08:00.061162: done, saving took 1.05 seconds 
+2022-04-10 20:08:00.074353: done 
+2022-04-10 20:08:17.876787: This epoch took 511.217606 s
+ 
+2022-04-10 20:08:17.877050: 
+epoch:  68 
+2022-04-10 20:16:02.475337: train loss : 0.1051 
+2022-04-10 20:16:28.888537: validation loss: 0.1147 
+2022-04-10 20:16:28.889154: Average global foreground Dice: [0.6785, 0.8365] 
+2022-04-10 20:16:28.889242: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 20:16:29.290329: lr: 0.000144 
+2022-04-10 20:16:29.290522: saving scheduled checkpoint file... 
+2022-04-10 20:16:29.348368: saving checkpoint... 
+2022-04-10 20:16:30.164791: done, saving took 0.87 seconds 
+2022-04-10 20:16:30.178703: done 
+2022-04-10 20:16:30.212388: saving checkpoint... 
+2022-04-10 20:16:31.050560: done, saving took 0.87 seconds 
+2022-04-10 20:16:48.825418: This epoch took 510.948266 s
+ 
+2022-04-10 20:16:48.825706: 
+epoch:  69 
+2022-04-10 20:24:33.085171: train loss : 0.1041 
+2022-04-10 20:24:59.530747: validation loss: 0.1181 
+2022-04-10 20:24:59.531362: Average global foreground Dice: [0.6885, 0.8294] 
+2022-04-10 20:24:59.531446: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 20:24:59.932734: lr: 0.00014 
+2022-04-10 20:24:59.932929: saving scheduled checkpoint file... 
+2022-04-10 20:24:59.989264: saving checkpoint... 
+2022-04-10 20:25:00.845014: done, saving took 0.91 seconds 
+2022-04-10 20:25:00.858279: done 
+2022-04-10 20:25:00.888326: saving checkpoint... 
+2022-04-10 20:25:01.755199: done, saving took 0.90 seconds 
+2022-04-10 20:25:32.849222: This epoch took 524.023408 s
+ 
+2022-04-10 20:25:32.849483: 
+epoch:  70 
+2022-04-10 20:33:17.221514: train loss : 0.1111 
+2022-04-10 20:33:43.645937: validation loss: 0.1109 
+2022-04-10 20:33:43.646557: Average global foreground Dice: [0.6816, 0.8459] 
+2022-04-10 20:33:43.646643: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 20:33:44.048525: lr: 0.000136 
+2022-04-10 20:33:44.048718: saving scheduled checkpoint file... 
+2022-04-10 20:33:44.105591: saving checkpoint... 
+2022-04-10 20:33:44.918302: done, saving took 0.87 seconds 
+2022-04-10 20:33:44.932221: done 
+2022-04-10 20:33:44.965675: saving checkpoint... 
+2022-04-10 20:33:45.807418: done, saving took 0.88 seconds 
+2022-04-10 20:34:01.503301: This epoch took 508.653713 s
+ 
+2022-04-10 20:34:01.503607: 
+epoch:  71 
+2022-04-10 20:41:45.190751: train loss : 0.1071 
+2022-04-10 20:42:11.630767: validation loss: 0.1136 
+2022-04-10 20:42:11.631518: Average global foreground Dice: [0.6814, 0.8316] 
+2022-04-10 20:42:11.631627: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 20:42:12.045070: lr: 0.000132 
+2022-04-10 20:42:12.045274: saving scheduled checkpoint file... 
+2022-04-10 20:42:12.102142: saving checkpoint... 
+2022-04-10 20:42:12.921462: done, saving took 0.88 seconds 
+2022-04-10 20:42:12.934599: done 
+2022-04-10 20:42:12.966705: saving checkpoint... 
+2022-04-10 20:42:13.805837: done, saving took 0.87 seconds 
+2022-04-10 20:42:30.006332: This epoch took 508.502621 s
+ 
+2022-04-10 20:42:30.006595: 
+epoch:  72 
+2022-04-10 20:50:14.361686: train loss : 0.1033 
+2022-04-10 20:50:40.742469: validation loss: 0.1165 
+2022-04-10 20:50:40.743101: Average global foreground Dice: [0.6795, 0.8391] 
+2022-04-10 20:50:40.743203: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 20:50:41.144576: lr: 0.000128 
+2022-04-10 20:50:41.144771: saving scheduled checkpoint file... 
+2022-04-10 20:50:41.200487: saving checkpoint... 
+2022-04-10 20:50:42.080346: done, saving took 0.94 seconds 
+2022-04-10 20:50:42.098068: done 
+2022-04-10 20:50:42.138490: saving checkpoint... 
+2022-04-10 20:50:43.029720: done, saving took 0.93 seconds 
+2022-04-10 20:51:06.517043: This epoch took 516.510338 s
+ 
+2022-04-10 20:51:06.517421: 
+epoch:  73 
+2022-04-10 20:58:50.825136: train loss : 0.1090 
+2022-04-10 20:59:17.230968: validation loss: 0.1088 
+2022-04-10 20:59:17.231548: Average global foreground Dice: [0.6845, 0.8406] 
+2022-04-10 20:59:17.231637: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 20:59:17.644915: lr: 0.000124 
+2022-04-10 20:59:17.645104: saving scheduled checkpoint file... 
+2022-04-10 20:59:17.700877: saving checkpoint... 
+2022-04-10 20:59:18.517768: done, saving took 0.87 seconds 
+2022-04-10 20:59:18.530975: done 
+2022-04-10 20:59:18.561591: saving checkpoint... 
+2022-04-10 20:59:19.398534: done, saving took 0.87 seconds 
+2022-04-10 20:59:44.137997: This epoch took 517.620464 s
+ 
+2022-04-10 20:59:44.138342: 
+epoch:  74 
+2022-04-10 21:07:28.487542: train loss : 0.1029 
+2022-04-10 21:07:54.958955: validation loss: 0.1067 
+2022-04-10 21:07:54.959582: Average global foreground Dice: [0.6802, 0.8535] 
+2022-04-10 21:07:54.959670: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 21:07:55.392074: lr: 0.000119 
+2022-04-10 21:07:55.392340: saving scheduled checkpoint file... 
+2022-04-10 21:07:55.449190: saving checkpoint... 
+2022-04-10 21:07:56.360561: done, saving took 0.97 seconds 
+2022-04-10 21:07:56.373199: done 
+2022-04-10 21:07:56.401571: saving checkpoint... 
+2022-04-10 21:07:57.380114: done, saving took 1.01 seconds 
+2022-04-10 21:08:13.644437: This epoch took 509.505987 s
+ 
+2022-04-10 21:08:13.644716: 
+epoch:  75 
+2022-04-10 21:15:58.329515: train loss : 0.1083 
+2022-04-10 21:16:24.757826: validation loss: 0.1105 
+2022-04-10 21:16:24.758451: Average global foreground Dice: [0.6787, 0.8204] 
+2022-04-10 21:16:24.758538: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 21:16:25.167104: lr: 0.000115 
+2022-04-10 21:16:25.167304: saving scheduled checkpoint file... 
+2022-04-10 21:16:25.222942: saving checkpoint... 
+2022-04-10 21:16:26.209396: done, saving took 1.04 seconds 
+2022-04-10 21:16:26.222431: done 
+2022-04-10 21:16:43.684899: This epoch took 510.040067 s
+ 
+2022-04-10 21:16:43.685183: 
+epoch:  76 
+2022-04-10 21:24:27.664006: train loss : 0.1067 
+2022-04-10 21:24:54.058547: validation loss: 0.1184 
+2022-04-10 21:24:54.059151: Average global foreground Dice: [0.6864, 0.826] 
+2022-04-10 21:24:54.059239: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 21:24:54.463619: lr: 0.000112 
+2022-04-10 21:24:54.463814: saving scheduled checkpoint file... 
+2022-04-10 21:24:54.521375: saving checkpoint... 
+2022-04-10 21:24:55.419768: done, saving took 0.96 seconds 
+2022-04-10 21:24:55.433089: done 
+2022-04-10 21:25:43.506177: This epoch took 539.820878 s
+ 
+2022-04-10 21:25:43.506477: 
+epoch:  77 
+2022-04-10 21:33:27.970421: train loss : 0.1038 
+2022-04-10 21:33:54.375484: validation loss: 0.1003 
+2022-04-10 21:33:54.376197: Average global foreground Dice: [0.6903, 0.8447] 
+2022-04-10 21:33:54.376287: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 21:33:54.780878: lr: 0.000108 
+2022-04-10 21:33:54.781075: saving scheduled checkpoint file... 
+2022-04-10 21:33:54.838813: saving checkpoint... 
+2022-04-10 21:33:55.687676: done, saving took 0.91 seconds 
+2022-04-10 21:33:55.708688: done 
+2022-04-10 21:33:55.754733: saving checkpoint... 
+2022-04-10 21:33:56.715688: done, saving took 1.01 seconds 
+2022-04-10 21:34:13.951702: This epoch took 510.445126 s
+ 
+2022-04-10 21:34:13.951967: 
+epoch:  78 
+2022-04-10 21:41:58.402602: train loss : 0.1055 
+2022-04-10 21:42:24.836214: validation loss: 0.1130 
+2022-04-10 21:42:24.836861: Average global foreground Dice: [0.6769, 0.8276] 
+2022-04-10 21:42:24.836946: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 21:42:25.251900: lr: 0.000104 
+2022-04-10 21:42:25.252105: saving scheduled checkpoint file... 
+2022-04-10 21:42:25.307953: saving checkpoint... 
+2022-04-10 21:42:26.306599: done, saving took 1.05 seconds 
+2022-04-10 21:42:26.320059: done 
+2022-04-10 21:42:48.013165: This epoch took 514.061085 s
+ 
+2022-04-10 21:42:48.013431: 
+epoch:  79 
+2022-04-10 21:50:31.912853: train loss : 0.1071 
+2022-04-10 21:50:58.372591: validation loss: 0.1196 
+2022-04-10 21:50:58.373209: Average global foreground Dice: [0.6843, 0.8407] 
+2022-04-10 21:50:58.373297: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 21:50:58.779419: lr: 0.0001 
+2022-04-10 21:50:58.779611: saving scheduled checkpoint file... 
+2022-04-10 21:50:58.835851: saving checkpoint... 
+2022-04-10 21:50:59.710677: done, saving took 0.93 seconds 
+2022-04-10 21:50:59.723847: done 
+2022-04-10 21:51:16.350377: This epoch took 508.336843 s
+ 
+2022-04-10 21:51:16.350659: 
+epoch:  80 
+2022-04-10 21:59:00.787403: train loss : 0.1006 
+2022-04-10 21:59:27.253777: validation loss: 0.1196 
+2022-04-10 21:59:27.254355: Average global foreground Dice: [0.6714, 0.8375] 
+2022-04-10 21:59:27.254447: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 21:59:27.657805: lr: 9.6e-05 
+2022-04-10 21:59:27.657999: saving scheduled checkpoint file... 
+2022-04-10 21:59:27.714311: saving checkpoint... 
+2022-04-10 21:59:28.716470: done, saving took 1.06 seconds 
+2022-04-10 21:59:28.729259: done 
+2022-04-10 21:59:46.833516: This epoch took 510.482752 s
+ 
+2022-04-10 21:59:46.833987: 
+epoch:  81 
+2022-04-10 22:07:31.159551: train loss : 0.1011 
+2022-04-10 22:07:57.566154: validation loss: 0.1180 
+2022-04-10 22:07:57.566702: Average global foreground Dice: [0.668, 0.8251] 
+2022-04-10 22:07:57.566787: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 22:07:57.968626: lr: 9.2e-05 
+2022-04-10 22:07:57.968816: saving scheduled checkpoint file... 
+2022-04-10 22:07:58.024693: saving checkpoint... 
+2022-04-10 22:07:58.840169: done, saving took 0.87 seconds 
+2022-04-10 22:07:58.853130: done 
+2022-04-10 22:08:35.418130: This epoch took 528.584032 s
+ 
+2022-04-10 22:08:35.418416: 
+epoch:  82 
+2022-04-10 22:16:20.329367: train loss : 0.1013 
+2022-04-10 22:16:46.729283: validation loss: 0.1138 
+2022-04-10 22:16:46.729886: Average global foreground Dice: [0.673, 0.8545] 
+2022-04-10 22:16:46.729971: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 22:16:47.125348: lr: 8.8e-05 
+2022-04-10 22:16:47.125546: saving scheduled checkpoint file... 
+2022-04-10 22:16:47.183120: saving checkpoint... 
+2022-04-10 22:16:48.087715: done, saving took 0.96 seconds 
+2022-04-10 22:16:48.100435: done 
+2022-04-10 22:17:05.743816: This epoch took 510.325293 s
+ 
+2022-04-10 22:17:05.744102: 
+epoch:  83 
+2022-04-10 22:24:50.166980: train loss : 0.0978 
+2022-04-10 22:25:16.521012: validation loss: 0.1167 
+2022-04-10 22:25:16.521608: Average global foreground Dice: [0.6725, 0.8296] 
+2022-04-10 22:25:16.521699: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 22:25:16.928774: lr: 8.5e-05 
+2022-04-10 22:25:16.928965: saving scheduled checkpoint file... 
+2022-04-10 22:25:16.986414: saving checkpoint... 
+2022-04-10 22:25:17.804583: done, saving took 0.88 seconds 
+2022-04-10 22:25:17.818675: done 
+2022-04-10 22:25:35.578867: This epoch took 509.834507 s
+ 
+2022-04-10 22:25:35.579188: 
+epoch:  84 
+2022-04-10 22:33:19.554222: train loss : 0.1029 
+2022-04-10 22:33:45.962773: validation loss: 0.1183 
+2022-04-10 22:33:45.963323: Average global foreground Dice: [0.6749, 0.8491] 
+2022-04-10 22:33:45.963412: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 22:33:46.359160: lr: 8.1e-05 
+2022-04-10 22:33:46.359357: saving scheduled checkpoint file... 
+2022-04-10 22:33:46.415623: saving checkpoint... 
+2022-04-10 22:33:47.250269: done, saving took 0.89 seconds 
+2022-04-10 22:33:47.269721: done 
+2022-04-10 22:34:29.004492: This epoch took 533.425202 s
+ 
+2022-04-10 22:34:29.004789: 
+epoch:  85 
+2022-04-10 22:42:13.138645: train loss : 0.1068 
+2022-04-10 22:42:39.558180: validation loss: 0.1133 
+2022-04-10 22:42:39.558820: Average global foreground Dice: [0.6622, 0.8185] 
+2022-04-10 22:42:39.558923: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 22:42:39.955683: lr: 7.7e-05 
+2022-04-10 22:42:39.955875: saving scheduled checkpoint file... 
+2022-04-10 22:42:40.013230: saving checkpoint... 
+2022-04-10 22:42:40.833405: done, saving took 0.88 seconds 
+2022-04-10 22:42:40.846349: done 
+2022-04-10 22:42:57.502624: This epoch took 508.497737 s
+ 
+2022-04-10 22:42:57.502898: 
+epoch:  86 
+2022-04-10 22:50:41.896936: train loss : 0.0952 
+2022-04-10 22:51:08.347039: validation loss: 0.1119 
+2022-04-10 22:51:08.347669: Average global foreground Dice: [0.678, 0.8513] 
+2022-04-10 22:51:08.347837: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 22:51:08.743234: lr: 7.4e-05 
+2022-04-10 22:51:08.743423: saving scheduled checkpoint file... 
+2022-04-10 22:51:08.799773: saving checkpoint... 
+2022-04-10 22:51:09.625768: done, saving took 0.88 seconds 
+2022-04-10 22:51:09.640283: done 
+2022-04-10 22:51:27.745762: This epoch took 510.242764 s
+ 
+2022-04-10 22:51:27.746026: 
+epoch:  87 
+2022-04-10 22:59:12.076954: train loss : 0.1028 
+2022-04-10 22:59:38.494577: validation loss: 0.1176 
+2022-04-10 22:59:38.495210: Average global foreground Dice: [0.6747, 0.8458] 
+2022-04-10 22:59:38.495412: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 22:59:38.890906: lr: 7e-05 
+2022-04-10 22:59:38.891107: saving scheduled checkpoint file... 
+2022-04-10 22:59:38.947201: saving checkpoint... 
+2022-04-10 22:59:39.993511: done, saving took 1.10 seconds 
+2022-04-10 22:59:40.007102: done 
+2022-04-10 22:59:57.703154: This epoch took 509.957018 s
+ 
+2022-04-10 22:59:57.703449: 
+epoch:  88 
+2022-04-10 23:07:41.957302: train loss : 0.0994 
+2022-04-10 23:08:08.367649: validation loss: 0.1117 
+2022-04-10 23:08:08.368330: Average global foreground Dice: [0.6688, 0.8499] 
+2022-04-10 23:08:08.368418: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 23:08:08.766754: lr: 6.7e-05 
+2022-04-10 23:08:08.766953: saving scheduled checkpoint file... 
+2022-04-10 23:08:08.824034: saving checkpoint... 
+2022-04-10 23:08:09.799193: done, saving took 1.03 seconds 
+2022-04-10 23:08:09.813503: done 
+2022-04-10 23:08:26.260740: This epoch took 508.557184 s
+ 
+2022-04-10 23:08:26.261011: 
+epoch:  89 
+2022-04-10 23:16:10.343827: train loss : 0.1032 
+2022-04-10 23:16:36.766505: validation loss: 0.1197 
+2022-04-10 23:16:36.767093: Average global foreground Dice: [0.6688, 0.8224] 
+2022-04-10 23:16:36.767190: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 23:16:37.164858: lr: 6.3e-05 
+2022-04-10 23:16:37.165061: saving scheduled checkpoint file... 
+2022-04-10 23:16:37.222269: saving checkpoint... 
+2022-04-10 23:16:38.031365: done, saving took 0.87 seconds 
+2022-04-10 23:16:38.045878: done 
+2022-04-10 23:16:54.738627: This epoch took 508.477517 s
+ 
+2022-04-10 23:16:54.738901: 
+epoch:  90 
+2022-04-10 23:24:38.980242: train loss : 0.1039 
+2022-04-10 23:25:05.393491: validation loss: 0.1023 
+2022-04-10 23:25:05.394150: Average global foreground Dice: [0.6921, 0.8579] 
+2022-04-10 23:25:05.394243: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 23:25:05.808189: lr: 6e-05 
+2022-04-10 23:25:05.808378: saving scheduled checkpoint file... 
+2022-04-10 23:25:05.866400: saving checkpoint... 
+2022-04-10 23:25:06.703244: done, saving took 0.89 seconds 
+2022-04-10 23:25:06.718752: done 
+2022-04-10 23:25:24.890051: This epoch took 510.151049 s
+ 
+2022-04-10 23:25:24.890347: 
+epoch:  91 
+2022-04-10 23:33:08.572079: train loss : 0.1002 
+2022-04-10 23:33:35.014786: validation loss: 0.1068 
+2022-04-10 23:33:35.015433: Average global foreground Dice: [0.6819, 0.8442] 
+2022-04-10 23:33:35.015542: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 23:33:35.416020: lr: 5.7e-05 
+2022-04-10 23:33:35.416218: saving scheduled checkpoint file... 
+2022-04-10 23:33:35.473092: saving checkpoint... 
+2022-04-10 23:33:36.318601: done, saving took 0.90 seconds 
+2022-04-10 23:33:36.339875: done 
+2022-04-10 23:33:36.388475: saving checkpoint... 
+2022-04-10 23:33:37.408233: done, saving took 1.07 seconds 
+2022-04-10 23:33:57.774623: This epoch took 512.884171 s
+ 
+2022-04-10 23:33:57.774908: 
+epoch:  92 
+2022-04-10 23:41:42.513762: train loss : 0.0971 
+2022-04-10 23:42:08.950256: validation loss: 0.1189 
+2022-04-10 23:42:08.950849: Average global foreground Dice: [0.6669, 0.8311] 
+2022-04-10 23:42:08.950937: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 23:42:09.361015: lr: 5.4e-05 
+2022-04-10 23:42:09.361218: saving scheduled checkpoint file... 
+2022-04-10 23:42:09.416616: saving checkpoint... 
+2022-04-10 23:42:10.455077: done, saving took 1.09 seconds 
+2022-04-10 23:42:10.468337: done 
+2022-04-10 23:42:29.419454: This epoch took 511.644444 s
+ 
+2022-04-10 23:42:29.419739: 
+epoch:  93 
+2022-04-10 23:50:13.616708: train loss : 0.0994 
+2022-04-10 23:50:40.072582: validation loss: 0.1222 
+2022-04-10 23:50:40.073347: Average global foreground Dice: [0.6704, 0.8395] 
+2022-04-10 23:50:40.073437: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 23:50:40.474370: lr: 5.1e-05 
+2022-04-10 23:50:40.474565: saving scheduled checkpoint file... 
+2022-04-10 23:50:40.531165: saving checkpoint... 
+2022-04-10 23:50:41.417615: done, saving took 0.94 seconds 
+2022-04-10 23:50:41.430809: done 
+2022-04-10 23:51:01.087107: This epoch took 511.667259 s
+ 
+2022-04-10 23:51:01.087400: 
+epoch:  94 
+2022-04-10 23:58:45.297549: train loss : 0.0983 
+2022-04-10 23:59:11.734450: validation loss: 0.1203 
+2022-04-10 23:59:11.735062: Average global foreground Dice: [0.6508, 0.849] 
+2022-04-10 23:59:11.735160: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-10 23:59:12.134044: lr: 4.8e-05 
+2022-04-10 23:59:12.134236: saving scheduled checkpoint file... 
+2022-04-10 23:59:12.191916: saving checkpoint... 
+2022-04-10 23:59:13.012859: done, saving took 0.88 seconds 
+2022-04-10 23:59:13.025613: done 
+2022-04-10 23:59:44.263065: This epoch took 523.175564 s
+ 
+2022-04-10 23:59:44.263381: 
+epoch:  95 
+2022-04-11 00:07:28.551691: train loss : 0.0973 
+2022-04-11 00:07:54.956961: validation loss: 0.1107 
+2022-04-11 00:07:54.957588: Average global foreground Dice: [0.6677, 0.8522] 
+2022-04-11 00:07:54.957672: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 00:07:55.359202: lr: 4.5e-05 
+2022-04-11 00:07:55.359390: saving scheduled checkpoint file... 
+2022-04-11 00:07:55.417162: saving checkpoint... 
+2022-04-11 00:07:56.294970: done, saving took 0.94 seconds 
+2022-04-11 00:07:56.310632: done 
+2022-04-11 00:08:14.391358: This epoch took 510.127879 s
+ 
+2022-04-11 00:08:14.391646: 
+epoch:  96 
+2022-04-11 00:15:58.379297: train loss : 0.1001 
+2022-04-11 00:16:24.829058: validation loss: 0.1118 
+2022-04-11 00:16:24.829665: Average global foreground Dice: [0.6776, 0.804] 
+2022-04-11 00:16:24.829749: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 00:16:25.232229: lr: 4.2e-05 
+2022-04-11 00:16:25.232417: saving scheduled checkpoint file... 
+2022-04-11 00:16:25.287850: saving checkpoint... 
+2022-04-11 00:16:26.116492: done, saving took 0.88 seconds 
+2022-04-11 00:16:26.129269: done 
+2022-04-11 00:16:47.828458: This epoch took 513.436691 s
+ 
+2022-04-11 00:16:47.828829: 
+epoch:  97 
+2022-04-11 00:24:31.950984: train loss : 0.0986 
+2022-04-11 00:24:58.329758: validation loss: 0.1188 
+2022-04-11 00:24:58.330380: Average global foreground Dice: [0.6682, 0.8525] 
+2022-04-11 00:24:58.330466: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 00:24:58.732141: lr: 3.9e-05 
+2022-04-11 00:24:58.732334: saving scheduled checkpoint file... 
+2022-04-11 00:24:58.790140: saving checkpoint... 
+2022-04-11 00:24:59.646938: done, saving took 0.91 seconds 
+2022-04-11 00:24:59.660223: done 
+2022-04-11 00:25:24.683101: This epoch took 516.854163 s
+ 
+2022-04-11 00:25:24.683408: 
+epoch:  98 
+2022-04-11 00:33:08.420400: train loss : 0.0902 
+2022-04-11 00:33:34.909183: validation loss: 0.1224 
+2022-04-11 00:33:34.909782: Average global foreground Dice: [0.6841, 0.8114] 
+2022-04-11 00:33:34.909868: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 00:33:35.310576: lr: 3.6e-05 
+2022-04-11 00:33:35.310761: saving scheduled checkpoint file... 
+2022-04-11 00:33:35.367307: saving checkpoint... 
+2022-04-11 00:33:36.389843: done, saving took 1.08 seconds 
+2022-04-11 00:33:36.402301: done 
+2022-04-11 00:33:53.173640: This epoch took 508.490132 s
+ 
+2022-04-11 00:33:53.173917: 
+epoch:  99 
+2022-04-11 00:41:37.226401: train loss : 0.1021 
+2022-04-11 00:42:03.608253: validation loss: 0.1000 
+2022-04-11 00:42:03.608792: Average global foreground Dice: [0.676, 0.8473] 
+2022-04-11 00:42:03.608874: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 00:42:04.008083: lr: 3.4e-05 
+2022-04-11 00:42:04.008273: saving scheduled checkpoint file... 
+2022-04-11 00:42:04.065178: saving checkpoint... 
+2022-04-11 00:42:05.096664: done, saving took 1.09 seconds 
+2022-04-11 00:42:05.108881: done 
+2022-04-11 00:42:24.771225: This epoch took 511.597147 s
+ 
+2022-04-11 00:42:24.771495: 
+epoch:  100 
+2022-04-11 00:50:08.618661: train loss : 0.1002 
+2022-04-11 00:50:35.018148: validation loss: 0.1056 
+2022-04-11 00:50:35.018898: Average global foreground Dice: [0.666, 0.8368] 
+2022-04-11 00:50:35.019013: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 00:50:35.432341: lr: 3.1e-05 
+2022-04-11 00:50:35.432531: saving scheduled checkpoint file... 
+2022-04-11 00:50:35.489046: saving checkpoint... 
+2022-04-11 00:50:36.532067: done, saving took 1.10 seconds 
+2022-04-11 00:50:36.544300: done 
+2022-04-11 00:50:54.964104: This epoch took 510.192511 s
+ 
+2022-04-11 00:50:54.964399: 
+epoch:  101 
+2022-04-11 00:58:38.495480: train loss : 0.0997 
+2022-04-11 00:59:04.898658: validation loss: 0.1131 
+2022-04-11 00:59:04.899313: Average global foreground Dice: [0.6778, 0.8556] 
+2022-04-11 00:59:04.899400: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 00:59:05.301531: lr: 2.9e-05 
+2022-04-11 00:59:05.301713: saving scheduled checkpoint file... 
+2022-04-11 00:59:05.358215: saving checkpoint... 
+2022-04-11 00:59:06.325876: done, saving took 1.02 seconds 
+2022-04-11 00:59:06.339094: done 
+2022-04-11 00:59:26.638718: This epoch took 511.674201 s
+ 
+2022-04-11 00:59:26.639078: 
+epoch:  102 
+2022-04-11 01:07:10.752940: train loss : 0.0990 
+2022-04-11 01:07:37.122280: validation loss: 0.1250 
+2022-04-11 01:07:37.122834: Average global foreground Dice: [0.6707, 0.8374] 
+2022-04-11 01:07:37.122919: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 01:07:37.525373: lr: 2.6e-05 
+2022-04-11 01:07:37.525567: saving scheduled checkpoint file... 
+2022-04-11 01:07:37.581885: saving checkpoint... 
+2022-04-11 01:07:38.411311: done, saving took 0.89 seconds 
+2022-04-11 01:07:38.424269: done 
+2022-04-11 01:07:56.737182: This epoch took 510.097934 s
+ 
+2022-04-11 01:07:56.737455: 
+epoch:  103 
+2022-04-11 01:15:40.390130: train loss : 0.0967 
+2022-04-11 01:16:06.785503: validation loss: 0.1075 
+2022-04-11 01:16:06.786162: Average global foreground Dice: [0.6584, 0.8374] 
+2022-04-11 01:16:06.786257: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 01:16:07.186196: lr: 2.4e-05 
+2022-04-11 01:16:07.186376: saving scheduled checkpoint file... 
+2022-04-11 01:16:07.245436: saving checkpoint... 
+2022-04-11 01:16:08.223803: done, saving took 1.04 seconds 
+2022-04-11 01:16:08.237586: done 
+2022-04-11 01:16:26.410596: This epoch took 509.673037 s
+ 
+2022-04-11 01:16:26.410898: 
+epoch:  104 
+2022-04-11 01:24:10.693968: train loss : 0.0962 
+2022-04-11 01:24:37.085784: validation loss: 0.1174 
+2022-04-11 01:24:37.086400: Average global foreground Dice: [0.6659, 0.8253] 
+2022-04-11 01:24:37.086489: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 01:24:37.489119: lr: 2.2e-05 
+2022-04-11 01:24:37.489316: saving scheduled checkpoint file... 
+2022-04-11 01:24:37.546186: saving checkpoint... 
+2022-04-11 01:24:38.549479: done, saving took 1.06 seconds 
+2022-04-11 01:24:38.562896: done 
+2022-04-11 01:24:58.902035: This epoch took 512.491037 s
+ 
+2022-04-11 01:24:58.902358: 
+epoch:  105 
+2022-04-11 01:32:43.345103: train loss : 0.0967 
+2022-04-11 01:33:09.745736: validation loss: 0.1190 
+2022-04-11 01:33:09.746476: Average global foreground Dice: [0.6618, 0.8518] 
+2022-04-11 01:33:09.746567: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 01:33:10.150225: lr: 2e-05 
+2022-04-11 01:33:10.150417: saving scheduled checkpoint file... 
+2022-04-11 01:33:10.207119: saving checkpoint... 
+2022-04-11 01:33:11.022336: done, saving took 0.87 seconds 
+2022-04-11 01:33:11.035040: done 
+2022-04-11 01:33:28.587493: This epoch took 509.685036 s
+ 
+2022-04-11 01:33:28.587766: 
+epoch:  106 
+2022-04-11 01:41:12.666342: train loss : 0.0958 
+2022-04-11 01:41:39.122420: validation loss: 0.1110 
+2022-04-11 01:41:39.123175: Average global foreground Dice: [0.672, 0.8456] 
+2022-04-11 01:41:39.123265: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 01:41:39.549054: lr: 1.8e-05 
+2022-04-11 01:41:39.549254: saving scheduled checkpoint file... 
+2022-04-11 01:41:39.605859: saving checkpoint... 
+2022-04-11 01:41:40.427374: done, saving took 0.88 seconds 
+2022-04-11 01:41:40.440282: done 
+2022-04-11 01:42:17.150831: This epoch took 528.562963 s
+ 
+2022-04-11 01:42:17.151115: 
+epoch:  107 
+2022-04-11 01:50:01.437896: train loss : 0.1020 
+2022-04-11 01:50:27.837887: validation loss: 0.1066 
+2022-04-11 01:50:27.838478: Average global foreground Dice: [0.6706, 0.8587] 
+2022-04-11 01:50:27.838567: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 01:50:28.240228: lr: 1.6e-05 
+2022-04-11 01:50:28.240415: saving scheduled checkpoint file... 
+2022-04-11 01:50:28.297682: saving checkpoint... 
+2022-04-11 01:50:29.306407: done, saving took 1.07 seconds 
+2022-04-11 01:50:29.319225: done 
+2022-04-11 01:50:52.725280: This epoch took 515.574038 s
+ 
+2022-04-11 01:50:52.725604: 
+epoch:  108 
+2022-04-11 01:58:37.514179: train loss : 0.1048 
+2022-04-11 01:59:03.925513: validation loss: 0.0996 
+2022-04-11 01:59:03.926161: Average global foreground Dice: [0.6717, 0.8588] 
+2022-04-11 01:59:03.926348: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 01:59:04.330530: lr: 1.4e-05 
+2022-04-11 01:59:04.330709: saving scheduled checkpoint file... 
+2022-04-11 01:59:04.386258: saving checkpoint... 
+2022-04-11 01:59:05.280504: done, saving took 0.95 seconds 
+2022-04-11 01:59:05.294529: done 
+2022-04-11 01:59:23.206556: This epoch took 510.480844 s
+ 
+2022-04-11 01:59:23.206860: 
+epoch:  109 
+2022-04-11 02:07:07.691472: train loss : 0.1003 
+2022-04-11 02:07:34.116198: validation loss: 0.1191 
+2022-04-11 02:07:34.116776: Average global foreground Dice: [0.6779, 0.8419] 
+2022-04-11 02:07:34.116864: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 02:07:34.520701: lr: 1.2e-05 
+2022-04-11 02:07:34.520870: saving scheduled checkpoint file... 
+2022-04-11 02:07:34.577405: saving checkpoint... 
+2022-04-11 02:07:35.395641: done, saving took 0.87 seconds 
+2022-04-11 02:07:35.408465: done 
+2022-04-11 02:07:56.198654: This epoch took 512.991689 s
+ 
+2022-04-11 02:07:56.198972: 
+epoch:  110 
+2022-04-11 02:15:40.677786: train loss : 0.0900 
+2022-04-11 02:16:07.066796: validation loss: 0.1121 
+2022-04-11 02:16:07.067431: Average global foreground Dice: [0.6768, 0.8574] 
+2022-04-11 02:16:07.067522: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 02:16:07.470771: lr: 1.1e-05 
+2022-04-11 02:16:07.470952: saving scheduled checkpoint file... 
+2022-04-11 02:16:07.528092: saving checkpoint... 
+2022-04-11 02:16:08.348845: done, saving took 0.88 seconds 
+2022-04-11 02:16:08.362813: done 
+2022-04-11 02:16:27.957670: This epoch took 511.758583 s
+ 
+2022-04-11 02:16:27.957937: 
+epoch:  111 
+2022-04-11 02:24:12.534302: train loss : 0.0937 
+2022-04-11 02:24:38.922717: validation loss: 0.1108 
+2022-04-11 02:24:38.923304: Average global foreground Dice: [0.6711, 0.8324] 
+2022-04-11 02:24:38.923392: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 02:24:39.341762: lr: 9e-06 
+2022-04-11 02:24:39.341936: saving scheduled checkpoint file... 
+2022-04-11 02:24:39.401751: saving checkpoint... 
+2022-04-11 02:24:40.233243: done, saving took 0.89 seconds 
+2022-04-11 02:24:40.246188: done 
+2022-04-11 02:24:59.187464: This epoch took 511.229429 s
+ 
+2022-04-11 02:24:59.187735: 
+epoch:  112 
+2022-04-11 02:32:42.822770: train loss : 0.0990 
+2022-04-11 02:33:09.284059: validation loss: 0.1129 
+2022-04-11 02:33:09.284718: Average global foreground Dice: [0.6703, 0.8522] 
+2022-04-11 02:33:09.284807: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 02:33:09.692517: lr: 8e-06 
+2022-04-11 02:33:09.692719: saving scheduled checkpoint file... 
+2022-04-11 02:33:09.752110: saving checkpoint... 
+2022-04-11 02:33:10.768755: done, saving took 1.08 seconds 
+2022-04-11 02:33:10.782397: done 
+2022-04-11 02:33:29.761414: This epoch took 510.573584 s
+ 
+2022-04-11 02:33:29.761697: 
+epoch:  113 
+2022-04-11 02:41:13.891632: train loss : 0.0943 
+2022-04-11 02:41:40.315355: validation loss: 0.1016 
+2022-04-11 02:41:40.316029: Average global foreground Dice: [0.6762, 0.8627] 
+2022-04-11 02:41:40.316132: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 02:41:40.719229: lr: 7e-06 
+2022-04-11 02:41:40.719421: saving scheduled checkpoint file... 
+2022-04-11 02:41:40.776216: saving checkpoint... 
+2022-04-11 02:41:41.793904: done, saving took 1.07 seconds 
+2022-04-11 02:41:41.807141: done 
+2022-04-11 02:41:41.836925: saving checkpoint... 
+2022-04-11 02:41:42.808490: done, saving took 1.00 seconds 
+2022-04-11 02:42:09.560593: This epoch took 519.798784 s
+ 
+2022-04-11 02:42:09.560908: 
+epoch:  114 
+2022-04-11 02:49:53.667938: train loss : 0.0916 
+2022-04-11 02:50:20.091251: validation loss: 0.1308 
+2022-04-11 02:50:20.091951: Average global foreground Dice: [0.6721, 0.8363] 
+2022-04-11 02:50:20.092038: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 02:50:20.497740: lr: 6e-06 
+2022-04-11 02:50:20.497926: saving scheduled checkpoint file... 
+2022-04-11 02:50:20.553823: saving checkpoint... 
+2022-04-11 02:50:21.590157: done, saving took 1.09 seconds 
+2022-04-11 02:50:21.603029: done 
+2022-04-11 02:50:44.950327: This epoch took 515.389312 s
+ 
+2022-04-11 02:50:44.950629: 
+epoch:  115 
+2022-04-11 02:58:29.238728: train loss : 0.0989 
+2022-04-11 02:58:55.640378: validation loss: 0.0862 
+2022-04-11 02:58:55.640920: Average global foreground Dice: [0.6794, 0.8411] 
+2022-04-11 02:58:55.641003: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 02:58:56.046362: lr: 5e-06 
+2022-04-11 02:58:56.046554: saving scheduled checkpoint file... 
+2022-04-11 02:58:56.103735: saving checkpoint... 
+2022-04-11 02:58:57.014611: done, saving took 0.97 seconds 
+2022-04-11 02:58:57.027855: done 
+2022-04-11 02:59:18.129505: This epoch took 513.178772 s
+ 
+2022-04-11 02:59:18.129809: 
+epoch:  116 
+2022-04-11 03:07:02.338817: train loss : 0.0955 
+2022-04-11 03:07:28.774480: validation loss: 0.0984 
+2022-04-11 03:07:28.775194: Average global foreground Dice: [0.6729, 0.8408] 
+2022-04-11 03:07:28.775284: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 03:07:29.179642: lr: 4e-06 
+2022-04-11 03:07:29.179827: saving scheduled checkpoint file... 
+2022-04-11 03:07:29.236301: saving checkpoint... 
+2022-04-11 03:07:30.282170: done, saving took 1.10 seconds 
+2022-04-11 03:07:30.307718: done 
+2022-04-11 03:07:53.854745: This epoch took 515.724835 s
+ 
+2022-04-11 03:07:53.855031: 
+epoch:  117 
+2022-04-11 03:15:38.147197: train loss : 0.1001 
+2022-04-11 03:16:04.535587: validation loss: 0.1082 
+2022-04-11 03:16:04.536175: Average global foreground Dice: [0.6766, 0.8522] 
+2022-04-11 03:16:04.536261: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 03:16:04.940399: lr: 3e-06 
+2022-04-11 03:16:04.940575: saving scheduled checkpoint file... 
+2022-04-11 03:16:04.998541: saving checkpoint... 
+2022-04-11 03:16:06.022472: done, saving took 1.08 seconds 
+2022-04-11 03:16:06.034996: done 
+2022-04-11 03:16:06.064238: saving checkpoint... 
+2022-04-11 03:16:07.092155: done, saving took 1.06 seconds 
+2022-04-11 03:16:32.860877: This epoch took 519.005736 s
+ 
+2022-04-11 03:16:32.861167: 
+epoch:  118 
+2022-04-11 03:24:17.241937: train loss : 0.0953 
+2022-04-11 03:24:43.653858: validation loss: 0.1013 
+2022-04-11 03:24:43.654464: Average global foreground Dice: [0.6799, 0.8529] 
+2022-04-11 03:24:43.654552: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 03:24:44.059050: lr: 2e-06 
+2022-04-11 03:24:44.059263: saving scheduled checkpoint file... 
+2022-04-11 03:24:44.116023: saving checkpoint... 
+2022-04-11 03:24:45.028354: done, saving took 0.97 seconds 
+2022-04-11 03:24:45.040962: done 
+2022-04-11 03:24:45.070351: saving checkpoint... 
+2022-04-11 03:24:45.943426: done, saving took 0.90 seconds 
+2022-04-11 03:25:06.090891: This epoch took 513.229614 s
+ 
+2022-04-11 03:25:06.091197: 
+epoch:  119 
+2022-04-11 03:32:49.935872: train loss : 0.0902 
+2022-04-11 03:33:16.314792: validation loss: 0.1010 
+2022-04-11 03:33:16.315364: Average global foreground Dice: [0.677, 0.8553] 
+2022-04-11 03:33:16.315454: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 03:33:16.720098: lr: 1e-06 
+2022-04-11 03:33:16.720294: saving scheduled checkpoint file... 
+2022-04-11 03:33:16.777173: saving checkpoint... 
+2022-04-11 03:33:17.601157: done, saving took 0.88 seconds 
+2022-04-11 03:33:17.615427: done 
+2022-04-11 03:33:17.648793: saving checkpoint... 
+2022-04-11 03:33:18.499983: done, saving took 0.88 seconds 
+2022-04-11 03:33:43.146490: This epoch took 517.055192 s
+ 
+2022-04-11 03:33:43.146762: 
+epoch:  120 
+2022-04-11 03:41:27.493778: train loss : 0.0995 
+2022-04-11 03:41:53.864388: validation loss: 0.1036 
+2022-04-11 03:41:53.864993: Average global foreground Dice: [0.672, 0.8379] 
+2022-04-11 03:41:53.865082: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 03:41:54.271022: lr: 1e-06 
+2022-04-11 03:41:54.271216: saving scheduled checkpoint file... 
+2022-04-11 03:41:54.329515: saving checkpoint... 
+2022-04-11 03:41:55.278565: done, saving took 1.01 seconds 
+2022-04-11 03:41:55.291958: done 
+2022-04-11 03:42:16.711583: This epoch took 513.564715 s
+ 
+2022-04-11 03:42:16.711873: 
+epoch:  121 
+2022-04-11 03:50:00.528970: train loss : 0.0930 
+2022-04-11 03:50:26.907942: validation loss: 0.1102 
+2022-04-11 03:50:26.908743: Average global foreground Dice: [0.6683, 0.839] 
+2022-04-11 03:50:26.908829: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 03:50:27.325293: lr: 1e-06 
+2022-04-11 03:50:27.325490: saving scheduled checkpoint file... 
+2022-04-11 03:50:27.383005: saving checkpoint... 
+2022-04-11 03:50:28.426841: done, saving took 1.10 seconds 
+2022-04-11 03:50:28.443830: done 
+2022-04-11 03:50:50.745321: This epoch took 514.033344 s
+ 
+2022-04-11 03:50:50.745633: 
+epoch:  122 
+2022-04-11 03:58:34.703974: train loss : 0.0927 
+2022-04-11 03:59:01.127075: validation loss: 0.1155 
+2022-04-11 03:59:01.127673: Average global foreground Dice: [0.6688, 0.8489] 
+2022-04-11 03:59:01.127759: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 03:59:01.538339: lr: 0.0 
+2022-04-11 03:59:01.538524: saving scheduled checkpoint file... 
+2022-04-11 03:59:01.597607: saving checkpoint... 
+2022-04-11 03:59:02.523829: done, saving took 0.99 seconds 
+2022-04-11 03:59:02.537603: done 
+2022-04-11 03:59:31.505065: This epoch took 520.759331 s
+ 
+2022-04-11 03:59:31.505372: 
+epoch:  123 
+2022-04-11 04:07:16.063560: train loss : 0.0929 
+2022-04-11 04:07:42.469208: validation loss: 0.1100 
+2022-04-11 04:07:42.469787: Average global foreground Dice: [0.6818, 0.8564] 
+2022-04-11 04:07:42.469872: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 04:07:42.875737: lr: 0.0 
+2022-04-11 04:07:42.875914: saving scheduled checkpoint file... 
+2022-04-11 04:07:42.932545: saving checkpoint... 
+2022-04-11 04:07:43.988114: done, saving took 1.11 seconds 
+2022-04-11 04:07:44.006090: done 
+2022-04-11 04:08:06.132996: This epoch took 514.627523 s
+ 
+2022-04-11 04:08:06.133329: 
+epoch:  124 
+2022-04-11 04:15:50.707030: train loss : 0.0948 
+2022-04-11 04:16:17.195768: validation loss: 0.1099 
+2022-04-11 04:16:17.196349: Average global foreground Dice: [0.6777, 0.8503] 
+2022-04-11 04:16:17.196436: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-04-11 04:16:17.600261: lr: 0.0 
+2022-04-11 04:16:17.600437: saving scheduled checkpoint file... 
+2022-04-11 04:16:17.657469: saving checkpoint... 
+2022-04-11 04:16:18.493284: done, saving took 0.89 seconds 
+2022-04-11 04:16:18.508385: done 
+2022-04-11 04:16:18.542899: saving checkpoint... 
+2022-04-11 04:16:19.395175: done, saving took 0.89 seconds 
+2022-04-11 04:16:41.287956: This epoch took 515.154497 s
+ 
+2022-04-11 04:16:41.358451: saving checkpoint... 
+2022-04-11 04:16:42.239291: done, saving took 0.95 seconds 

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/debug.json

@@ -0,0 +1,106 @@
+{
+    "all_tr_losses": "[0.33880347, 0.23667154, 0.20787217, 0.17866741, 0.1688485, 0.15650244, 0.17122895, 0.16600037, 0.1662374, 0.14666808, 0.14834467, 0.14682831, 0.137326, 0.14589843, 0.14086902, 0.13520584, 0.13806397, 0.13688643, 0.14157186, 0.13476749, 0.13293879, 0.13792813, 0.12690659, 0.13224676, 0.12589811, 0.13101529, 0.121173665, 0.12743156, 0.12973718, 0.13155001, 0.12538257, 0.12880784, 0.11983098, 0.12570767, 0.12043454, 0.11550015, 0.12463293, 0.12751071, 0.12391978, 0.12015278, 0.122464985, 0.122478046, 0.11480512, 0.11710864, 0.11619793, 0.10997589, 0.121831164, 0.11592714, 0.11416377, 0.11117799, 0.12309044, 0.11643512, 0.111628205, 0.11246636, 0.11181403, 0.11948247, 0.11015611, 0.11349469, 0.10995185, 0.10290526, 0.11850418, 0.10616533, 0.10980081, 0.10753825, 0.10982819, 0.10064859, 0.10937555]",
+    "all_val_eval_metrics": "[0.24441803128538458, 0.3646200873534005, 0.4073836012941684, 0.5280979869389617, 0.5576282711651834, 0.5587112244661616, 0.616372627127902, 0.6399068696652392, 0.6740998923677397, 0.6733766897687383, 0.685950202676759, 0.6730891690792249, 0.6578517604180589, 0.6903512099612064, 0.6788547685772561, 0.6683383469176356, 0.6963632808493894, 0.706660171040598, 0.708526498652341, 0.7028626785285068, 0.6997068508123063, 0.671519035061525, 0.7146784462843951, 0.6687458102034116, 0.7247048220907977, 0.7174996675681652, 0.7223397463625885, 0.7132866945673728, 0.735415754668584, 0.7273902097652161, 0.7007612073468041, 0.7301585193118795, 0.7303057009425757, 0.7373153163204972, 0.719404512274642, 0.7335313201714158, 0.7210674690019635, 0.7344483733135108, 0.7313691909763442, 0.7268537501917084, 0.7311546278074248, 0.7148105526320396, 0.7228389278014391, 0.7419200118250955, 0.7401590515296674, 0.7414424569003581, 0.721725488437178, 0.736920930365019, 0.7281322335384874, 0.7534104846465608, 0.7341914504593126, 0.7376134606875642, 0.7306174240033567, 0.7329268753119342, 0.7365485371439937, 0.72699150625508, 0.7234505903870534, 0.7399885495692289, 0.7414867155021148, 0.7468206649655602, 0.744086049173363, 0.7281900657719618, 0.7319929288277218, 0.7345449775248943, 0.7430525868014256, 0.7610991095584116, 0.7367976474920181]",
+    "all_val_losses": "[0.26946953, 0.21931683, 0.20502658, 0.18677522, 0.17296845, 0.16148563, 0.15531741, 0.17155254, 0.1604264, 0.16581497, 0.15573883, 0.16156188, 0.15351474, 0.12892939, 0.15402336, 0.13484189, 0.14251389, 0.148972, 0.1342064, 0.13652013, 0.1399847, 0.12008429, 0.1167369, 0.1384284, 0.13724485, 0.12959942, 0.12515497, 0.14603548, 0.1266138, 0.13267235, 0.13720961, 0.1475334, 0.12584028, 0.1582998, 0.13734983, 0.15010451, 0.12458315, 0.1293641, 0.13138947, 0.15694696, 0.11656036, 0.12966883, 0.13392729, 0.12910281, 0.13069594, 0.12563862, 0.13533488, 0.1157934, 0.12913467, 0.13287137, 0.1307886, 0.12317187, 0.13162261, 0.134247, 0.114636295, 0.13997744, 0.11866906, 0.10774542, 0.118344985, 0.122042455, 0.15040416, 0.13680579, 0.124536514, 0.12542078, 0.11461933, 0.12392085, 0.12589943]",
+    "all_val_losses_tr_mode": "[]",
+    "also_val_in_tr_mode": "False",
+    "amp_grad_scaler": "<torch.cuda.amp.grad_scaler.GradScaler object at 0x7ff1c21d5940>",
+    "args": "Namespace(batch_size=2, config='./configs/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32.yaml', continue_training=False, crop_size=[64, 192, 192], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='4', fp32=False, hdfs_base='GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=125, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task008_HepaticVessel', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, vallatest=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)",
+    "base_num_features": "32",
+    "basic_generator_patch_size": "[178 308 233]",
+    "batch_dice": "True",
+    "batch_size": "2",
+    "best_MA_tr_loss_for_patience": "None",
+    "best_epoch_based_on_MA_tr_loss": "None",
+    "best_val_eval_criterion_MA": "None",
+    "classes": "[1, 2]",
+    "conv_per_stage": "2",
+    "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': False, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.2, 'do_scaling': True, 'scale_range': (0.7, 1.4), 'independent_scale_factor_for_each_axis': False, 'p_independent_scale_per_axis': 1, 'p_scale': 0.2, 'do_rotation': True, 'rotation_x': (-0.5235987755982988, 0.5235987755982988), 'rotation_y': (-0.5235987755982988, 0.5235987755982988), 'rotation_z': (-0.5235987755982988, 0.5235987755982988), 'rotation_p_per_axis': 1, 'p_rot': 0.2, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.7, 1.5), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, False)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': False, 'additive_brightness_p_per_sample': 0.15, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0.0, 'additive_brightness_sigma': 0.1, 'num_threads': 12, 'num_cached_per_thread': 2, 'patch_size_for_spatialtransform': array([ 64, 192, 192])}",
+    "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel",
+    "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]",
+    "deterministic": "False",
+    "disable_ds": "True",
+    "distribute_batch_size": "False",
+    "dl_tr": "<dataset_loading.DataLoader3D object at 0x7ff22a23ef60>",
+    "dl_val": "<dataset_loading.DataLoader3D object at 0x7ff22a248c50>",
+    "do_dummy_2D_aug": "False",
+    "ds_loss_weights": "[1. 0. 0. 0. 0.]",
+    "epoch": "67",
+    "experiment_name": "nnUNetTrainerV2_DDP",
+    "find_zero_weight_decay": "False",
+    "fold": "4",
+    "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetData_plans_v2.1_stage1",
+    "fp16": "True",
+    "global_batch_size": "16",
+    "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/gt_segmentations",
+    "inference_pad_border_mode": "constant",
+    "inference_pad_kwargs": "{'constant_values': 0}",
+    "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl', 4, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32', '/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel', True, 1, True, False, False, True)",
+    "initial_lr": "0.0003",
+    "input_size": "[64, 192, 192]",
+    "is_spatial_aug_only": "False",
+    "is_ssl": "False",
+    "layer_decay": "1.0",
+    "local_rank": "0",
+    "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/training_log_2022_5_19_21_33_38.txt",
+    "loss": "None",
+    "lr_scheduler": "<optimizers.lr_scheduler.LinearWarmupCosineAnnealingLR object at 0x7ff1c21b1860>",
+    "lr_scheduler_eps": "0.001",
+    "lr_scheduler_name": "warmup_cosine",
+    "lr_scheduler_patience": "30",
+    "lr_threshold": "1e-06",
+    "max_num_epochs": "125",
+    "min_lr": "0",
+    "min_region_size_per_class": "None",
+    "min_size_per_class": "None",
+    "model": "Generic_TransUNet_max_ppbp",
+    "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': True, 'vit_depth': 1, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}",
+    "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]",
+    "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]]",
+    "net_pool_per_axis": "[4, 5, 5]",
+    "normalization_schemes": "OrderedDict([(0, 'CT')])",
+    "num_batches_per_epoch": "250",
+    "num_classes": "3",
+    "num_input_channels": "1",
+    "num_val_batches_per_epoch": "50",
+    "online_eval_fn": "[]",
+    "online_eval_foreground_dc": "[]",
+    "online_eval_fp": "[]",
+    "online_eval_tp": "[]",
+    "only_keep_largest_connected_component": "None",
+    "optim_name": "adamw",
+    "optimizer": "AdamW (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    eps: 1e-08\n    initial_lr: 0.0003\n    lr: 0.00015204880106489267\n    weight_decay: 3e-05\n)",
+    "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4",
+    "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32",
+    "oversample_foreground_percent": "0.0",
+    "pad_all_sides": "None",
+    "patch_size": "[ 64 192 192]",
+    "patience": "50",
+    "pin_memory": "True",
+    "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/nnUNetPlansv2.1_plans_3D.pkl",
+    "reclip": "None",
+    "regions_class_order": "None",
+    "resume": "auto",
+    "save_best_checkpoint": "True",
+    "save_every": "1",
+    "save_final_checkpoint": "True",
+    "save_intermediate_checkpoints": "True",
+    "save_latest_only": "True",
+    "stage": "1",
+    "threeD": "True",
+    "tr_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7ff1eb24c710>",
+    "train_loss_MA": "None",
+    "train_loss_MA_alpha": "0.93",
+    "train_loss_MA_eps": "0.0005",
+    "transpose_backward": "[0, 1, 2]",
+    "transpose_forward": "[0, 1, 2]",
+    "unpack_data": "True",
+    "use_mask_for_norm": "OrderedDict([(0, False)])",
+    "use_progress_bar": "False",
+    "val_eval_criterion_MA": "None",
+    "val_eval_criterion_alpha": "0.9",
+    "val_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7ff1eb24c390>",
+    "warmup_epochs": "10",
+    "was_initialized": "True",
+    "weight_decay": "3e-05"
+}

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_best.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7aa266ff7a185c653bb63793d85a41205e156d57c092be397f71f569136a17bc
+size 496519226

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_best.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8206610de77e6d58ec32958396e4238a0b7a427ba091b1ac2b3cadfa42a66571
+size 188932

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_final_checkpoint.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed3d8700250477037c72f32e3f6746ed4c4853e59fc1166e0d4a78197beea8fc
+size 496529530

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_final_checkpoint.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8206610de77e6d58ec32958396e4238a0b7a427ba091b1ac2b3cadfa42a66571
+size 188932

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_latest.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96c5f9246437d9eb2ff06ce5c36bfff6397271fb895676c382d37d473de70fa8
+size 496529530

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_latest.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8206610de77e6d58ec32958396e4238a0b7a427ba091b1ac2b3cadfa42a66571
+size 188932

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/training_log_2022_5_19_21_33_38.txt

@@ -0,0 +1,769 @@
+Starting... 
+2022-05-19 21:33:38.214491: Using splits from existing split file: /opt/tiger/project/data/nnUNet_preprocessed/Task008_HepaticVessel/splits_final.pkl 
+2022-05-19 21:33:38.215025: The split file contains 5 splits. 
+2022-05-19 21:33:38.215077: Desired fold for training: 4 
+2022-05-19 21:33:38.215127: This split has 243 training and 60 validation cases. 
+2022-05-19 21:35:17.162176: TRAINING KEYS:
+ odict_keys(['hepaticvessel_001', 'hepaticvessel_002', 'hepaticvessel_004', 'hepaticvessel_005', 'hepaticvessel_007', 'hepaticvessel_010', 'hepaticvessel_011', 'hepaticvessel_013', 'hepaticvessel_016', 'hepaticvessel_018', 'hepaticvessel_019', 'hepaticvessel_022', 'hepaticvessel_023', 'hepaticvessel_025', 'hepaticvessel_026', 'hepaticvessel_027', 'hepaticvessel_028', 'hepaticvessel_029', 'hepaticvessel_030', 'hepaticvessel_031', 'hepaticvessel_039', 'hepaticvessel_040', 'hepaticvessel_042', 'hepaticvessel_044', 'hepaticvessel_046', 'hepaticvessel_050', 'hepaticvessel_051', 'hepaticvessel_052', 'hepaticvessel_053', 'hepaticvessel_058', 'hepaticvessel_061', 'hepaticvessel_062', 'hepaticvessel_065', 'hepaticvessel_066', 'hepaticvessel_068', 'hepaticvessel_071', 'hepaticvessel_072', 'hepaticvessel_077', 'hepaticvessel_078', 'hepaticvessel_079', 'hepaticvessel_080', 'hepaticvessel_081', 'hepaticvessel_082', 'hepaticvessel_083', 'hepaticvessel_084', 'hepaticvessel_085', 'hepaticvessel_086', 'hepaticvessel_087', 'hepaticvessel_090', 'hepaticvessel_091', 'hepaticvessel_093', 'hepaticvessel_096', 'hepaticvessel_097', 'hepaticvessel_098', 'hepaticvessel_100', 'hepaticvessel_102', 'hepaticvessel_103', 'hepaticvessel_104', 'hepaticvessel_110', 'hepaticvessel_112', 'hepaticvessel_116', 'hepaticvessel_117', 'hepaticvessel_125', 'hepaticvessel_127', 'hepaticvessel_128', 'hepaticvessel_129', 'hepaticvessel_131', 'hepaticvessel_132', 'hepaticvessel_133', 'hepaticvessel_137', 'hepaticvessel_138', 'hepaticvessel_139', 'hepaticvessel_140', 'hepaticvessel_141', 'hepaticvessel_142', 'hepaticvessel_144', 'hepaticvessel_145', 'hepaticvessel_146', 'hepaticvessel_147', 'hepaticvessel_148', 'hepaticvessel_150', 'hepaticvessel_151', 'hepaticvessel_157', 'hepaticvessel_160', 'hepaticvessel_161', 'hepaticvessel_163', 'hepaticvessel_164', 'hepaticvessel_165', 'hepaticvessel_166', 'hepaticvessel_167', 'hepaticvessel_171', 'hepaticvessel_172', 'hepaticvessel_175', 'hepaticvessel_177', 'hepaticvessel_178', 'hepaticvessel_179', 'hepaticvessel_183', 'hepaticvessel_184', 'hepaticvessel_185', 'hepaticvessel_186', 'hepaticvessel_192', 'hepaticvessel_193', 'hepaticvessel_194', 'hepaticvessel_196', 'hepaticvessel_197', 'hepaticvessel_198', 'hepaticvessel_199', 'hepaticvessel_201', 'hepaticvessel_203', 'hepaticvessel_206', 'hepaticvessel_209', 'hepaticvessel_210', 'hepaticvessel_213', 'hepaticvessel_214', 'hepaticvessel_215', 'hepaticvessel_217', 'hepaticvessel_218', 'hepaticvessel_219', 'hepaticvessel_221', 'hepaticvessel_222', 'hepaticvessel_224', 'hepaticvessel_225', 'hepaticvessel_227', 'hepaticvessel_229', 'hepaticvessel_230', 'hepaticvessel_231', 'hepaticvessel_233', 'hepaticvessel_234', 'hepaticvessel_235', 'hepaticvessel_236', 'hepaticvessel_238', 'hepaticvessel_240', 'hepaticvessel_242', 'hepaticvessel_244', 'hepaticvessel_245', 'hepaticvessel_246', 'hepaticvessel_248', 'hepaticvessel_255', 'hepaticvessel_256', 'hepaticvessel_258', 'hepaticvessel_259', 'hepaticvessel_263', 'hepaticvessel_265', 'hepaticvessel_266', 'hepaticvessel_268', 'hepaticvessel_269', 'hepaticvessel_270', 'hepaticvessel_272', 'hepaticvessel_274', 'hepaticvessel_275', 'hepaticvessel_279', 'hepaticvessel_280', 'hepaticvessel_281', 'hepaticvessel_282', 'hepaticvessel_284', 'hepaticvessel_285', 'hepaticvessel_286', 'hepaticvessel_287', 'hepaticvessel_289', 'hepaticvessel_290', 'hepaticvessel_291', 'hepaticvessel_294', 'hepaticvessel_296', 'hepaticvessel_297', 'hepaticvessel_299', 'hepaticvessel_300', 'hepaticvessel_303', 'hepaticvessel_307', 'hepaticvessel_309', 'hepaticvessel_314', 'hepaticvessel_318', 'hepaticvessel_319', 'hepaticvessel_320', 'hepaticvessel_321', 'hepaticvessel_322', 'hepaticvessel_323', 'hepaticvessel_324', 'hepaticvessel_325', 'hepaticvessel_327', 'hepaticvessel_329', 'hepaticvessel_330', 'hepaticvessel_332', 'hepaticvessel_333', 'hepaticvessel_336', 'hepaticvessel_338', 'hepaticvessel_340', 'hepaticvessel_341', 'hepaticvessel_345', 'hepaticvessel_348', 'hepaticvessel_349', 'hepaticvessel_353', 'hepaticvessel_356', 'hepaticvessel_358', 'hepaticvessel_359', 'hepaticvessel_361', 'hepaticvessel_362', 'hepaticvessel_363', 'hepaticvessel_367', 'hepaticvessel_368', 'hepaticvessel_369', 'hepaticvessel_371', 'hepaticvessel_374', 'hepaticvessel_375', 'hepaticvessel_382', 'hepaticvessel_383', 'hepaticvessel_384', 'hepaticvessel_385', 'hepaticvessel_388', 'hepaticvessel_389', 'hepaticvessel_391', 'hepaticvessel_398', 'hepaticvessel_400', 'hepaticvessel_401', 'hepaticvessel_406', 'hepaticvessel_407', 'hepaticvessel_408', 'hepaticvessel_409', 'hepaticvessel_411', 'hepaticvessel_412', 'hepaticvessel_416', 'hepaticvessel_419', 'hepaticvessel_420', 'hepaticvessel_422', 'hepaticvessel_423', 'hepaticvessel_424', 'hepaticvessel_427', 'hepaticvessel_429', 'hepaticvessel_431', 'hepaticvessel_433', 'hepaticvessel_434', 'hepaticvessel_437', 'hepaticvessel_438', 'hepaticvessel_440', 'hepaticvessel_441', 'hepaticvessel_442', 'hepaticvessel_443', 'hepaticvessel_444', 'hepaticvessel_445', 'hepaticvessel_446', 'hepaticvessel_447', 'hepaticvessel_453', 'hepaticvessel_456', 'hepaticvessel_458']) 
+2022-05-19 21:35:17.162389: VALIDATION KEYS:
+ odict_keys(['hepaticvessel_008', 'hepaticvessel_020', 'hepaticvessel_032', 'hepaticvessel_033', 'hepaticvessel_049', 'hepaticvessel_057', 'hepaticvessel_059', 'hepaticvessel_067', 'hepaticvessel_070', 'hepaticvessel_075', 'hepaticvessel_088', 'hepaticvessel_089', 'hepaticvessel_092', 'hepaticvessel_094', 'hepaticvessel_108', 'hepaticvessel_111', 'hepaticvessel_113', 'hepaticvessel_115', 'hepaticvessel_119', 'hepaticvessel_121', 'hepaticvessel_123', 'hepaticvessel_124', 'hepaticvessel_136', 'hepaticvessel_154', 'hepaticvessel_158', 'hepaticvessel_159', 'hepaticvessel_173', 'hepaticvessel_174', 'hepaticvessel_181', 'hepaticvessel_189', 'hepaticvessel_195', 'hepaticvessel_200', 'hepaticvessel_207', 'hepaticvessel_208', 'hepaticvessel_211', 'hepaticvessel_223', 'hepaticvessel_237', 'hepaticvessel_262', 'hepaticvessel_271', 'hepaticvessel_278', 'hepaticvessel_293', 'hepaticvessel_305', 'hepaticvessel_308', 'hepaticvessel_335', 'hepaticvessel_344', 'hepaticvessel_350', 'hepaticvessel_372', 'hepaticvessel_373', 'hepaticvessel_378', 'hepaticvessel_379', 'hepaticvessel_386', 'hepaticvessel_393', 'hepaticvessel_396', 'hepaticvessel_397', 'hepaticvessel_402', 'hepaticvessel_404', 'hepaticvessel_425', 'hepaticvessel_432', 'hepaticvessel_454', 'hepaticvessel_455']) 
+2022-05-19 21:35:25.556375: loading checkpoint /opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/fold_4/model_latest.model train= True 
+2022-05-19 21:35:26.553340: lr: 0.000148 
+2022-05-19 21:35:37.490287: 
+epoch:  67 
+2022-05-19 21:43:46.253406: train loss : 0.1096 
+2022-05-19 21:44:12.563544: validation loss: 0.1260 
+2022-05-19 21:44:12.564116: Average global foreground Dice: [0.6343, 0.8639] 
+2022-05-19 21:44:12.564205: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 21:44:12.977126: lr: 0.000148 
+2022-05-19 21:44:12.977350: saving scheduled checkpoint file... 
+2022-05-19 21:44:13.040467: saving checkpoint... 
+2022-05-19 21:44:13.845106: done, saving took 0.87 seconds 
+2022-05-19 21:44:13.853450: done 
+2022-05-19 21:44:25.647957: This epoch took 528.157012 s
+ 
+2022-05-19 21:44:25.648269: 
+epoch:  68 
+2022-05-19 21:52:17.157872: train loss : 0.1094 
+2022-05-19 21:52:43.458801: validation loss: 0.1116 
+2022-05-19 21:52:43.459407: Average global foreground Dice: [0.6421, 0.8485] 
+2022-05-19 21:52:43.459496: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 21:52:43.873522: lr: 0.000144 
+2022-05-19 21:52:43.873729: saving scheduled checkpoint file... 
+2022-05-19 21:52:43.926191: saving checkpoint... 
+2022-05-19 21:52:44.797381: done, saving took 0.92 seconds 
+2022-05-19 21:52:44.806211: done 
+2022-05-19 21:52:54.506831: This epoch took 508.858466 s
+ 
+2022-05-19 21:52:54.507107: 
+epoch:  69 
+2022-05-19 22:00:45.682789: train loss : 0.0999 
+2022-05-19 22:01:11.989265: validation loss: 0.1199 
+2022-05-19 22:01:11.989834: Average global foreground Dice: [0.6452, 0.8647] 
+2022-05-19 22:01:11.989922: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 22:01:12.394476: lr: 0.00014 
+2022-05-19 22:01:12.394692: saving scheduled checkpoint file... 
+2022-05-19 22:01:12.447033: saving checkpoint... 
+2022-05-19 22:01:13.409617: done, saving took 1.01 seconds 
+2022-05-19 22:01:13.418336: done 
+2022-05-19 22:01:13.449659: saving checkpoint... 
+2022-05-19 22:01:14.039936: done, saving took 0.62 seconds 
+2022-05-19 22:01:32.580511: This epoch took 518.073294 s
+ 
+2022-05-19 22:01:32.580812: 
+epoch:  70 
+2022-05-19 22:09:23.738479: train loss : 0.1008 
+2022-05-19 22:09:50.031369: validation loss: 0.1189 
+2022-05-19 22:09:50.031960: Average global foreground Dice: [0.6239, 0.8363] 
+2022-05-19 22:09:50.032046: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 22:09:50.440741: lr: 0.000136 
+2022-05-19 22:09:50.440928: saving scheduled checkpoint file... 
+2022-05-19 22:09:50.493215: saving checkpoint... 
+2022-05-19 22:09:51.395359: done, saving took 0.95 seconds 
+2022-05-19 22:09:51.404801: done 
+2022-05-19 22:10:08.475046: This epoch took 515.894123 s
+ 
+2022-05-19 22:10:08.475374: 
+epoch:  71 
+2022-05-19 22:17:59.751940: train loss : 0.1026 
+2022-05-19 22:18:26.083147: validation loss: 0.1145 
+2022-05-19 22:18:26.083803: Average global foreground Dice: [0.6274, 0.8535] 
+2022-05-19 22:18:26.083889: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 22:18:26.487436: lr: 0.000132 
+2022-05-19 22:18:26.487646: saving scheduled checkpoint file... 
+2022-05-19 22:18:26.540078: saving checkpoint... 
+2022-05-19 22:18:27.534005: done, saving took 1.05 seconds 
+2022-05-19 22:18:27.543228: done 
+2022-05-19 22:18:50.600508: This epoch took 522.125028 s
+ 
+2022-05-19 22:18:50.600816: 
+epoch:  72 
+2022-05-19 22:26:42.433744: train loss : 0.0976 
+2022-05-19 22:27:08.774569: validation loss: 0.1175 
+2022-05-19 22:27:08.775301: Average global foreground Dice: [0.6343, 0.8426] 
+2022-05-19 22:27:08.775386: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 22:27:09.178573: lr: 0.000128 
+2022-05-19 22:27:09.178778: saving scheduled checkpoint file... 
+2022-05-19 22:27:09.231058: saving checkpoint... 
+2022-05-19 22:27:10.137891: done, saving took 0.96 seconds 
+2022-05-19 22:27:10.147586: done 
+2022-05-19 22:28:05.570841: This epoch took 554.969912 s
+ 
+2022-05-19 22:28:05.571191: 
+epoch:  73 
+2022-05-19 22:35:57.063693: train loss : 0.1077 
+2022-05-19 22:36:23.386150: validation loss: 0.1141 
+2022-05-19 22:36:23.386826: Average global foreground Dice: [0.6311, 0.8464] 
+2022-05-19 22:36:23.386918: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 22:36:23.802702: lr: 0.000124 
+2022-05-19 22:36:23.802902: saving scheduled checkpoint file... 
+2022-05-19 22:36:23.854852: saving checkpoint... 
+2022-05-19 22:36:24.772301: done, saving took 0.97 seconds 
+2022-05-19 22:36:24.781648: done 
+2022-05-19 22:36:53.371611: This epoch took 527.800304 s
+ 
+2022-05-19 22:36:53.371925: 
+epoch:  74 
+2022-05-19 22:44:45.138723: train loss : 0.1085 
+2022-05-19 22:45:11.473521: validation loss: 0.1311 
+2022-05-19 22:45:11.474173: Average global foreground Dice: [0.6314, 0.8545] 
+2022-05-19 22:45:11.474261: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 22:45:11.882670: lr: 0.000119 
+2022-05-19 22:45:11.882869: saving scheduled checkpoint file... 
+2022-05-19 22:45:11.935653: saving checkpoint... 
+2022-05-19 22:45:12.894191: done, saving took 1.01 seconds 
+2022-05-19 22:45:12.903475: done 
+2022-05-19 22:45:39.162692: This epoch took 525.790651 s
+ 
+2022-05-19 22:45:39.163004: 
+epoch:  75 
+2022-05-19 22:53:30.929458: train loss : 0.1164 
+2022-05-19 22:53:57.243286: validation loss: 0.1266 
+2022-05-19 22:53:57.243943: Average global foreground Dice: [0.6344, 0.8554] 
+2022-05-19 22:53:57.244030: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 22:53:57.649608: lr: 0.000115 
+2022-05-19 22:53:57.649810: saving scheduled checkpoint file... 
+2022-05-19 22:53:57.701248: saving checkpoint... 
+2022-05-19 22:53:58.606626: done, saving took 0.96 seconds 
+2022-05-19 22:53:58.616650: done 
+2022-05-19 22:54:19.218716: This epoch took 520.055598 s
+ 
+2022-05-19 22:54:19.219012: 
+epoch:  76 
+2022-05-19 23:02:10.648831: train loss : 0.1008 
+2022-05-19 23:02:36.993830: validation loss: 0.1352 
+2022-05-19 23:02:36.994434: Average global foreground Dice: [0.6334, 0.8521] 
+2022-05-19 23:02:36.994521: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 23:02:37.404704: lr: 0.000112 
+2022-05-19 23:02:37.404987: saving scheduled checkpoint file... 
+2022-05-19 23:02:37.457659: saving checkpoint... 
+2022-05-19 23:02:38.377484: done, saving took 0.97 seconds 
+2022-05-19 23:02:38.386996: done 
+2022-05-19 23:05:15.718623: This epoch took 656.499499 s
+ 
+2022-05-19 23:05:15.718937: 
+epoch:  77 
+2022-05-19 23:13:07.240921: train loss : 0.1029 
+2022-05-19 23:13:33.559342: validation loss: 0.1186 
+2022-05-19 23:13:33.560020: Average global foreground Dice: [0.6441, 0.8557] 
+2022-05-19 23:13:33.560115: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 23:13:33.966158: lr: 0.000108 
+2022-05-19 23:13:33.966356: saving scheduled checkpoint file... 
+2022-05-19 23:13:34.018284: saving checkpoint... 
+2022-05-19 23:13:34.829733: done, saving took 0.86 seconds 
+2022-05-19 23:13:34.838025: done 
+2022-05-19 23:13:57.556308: This epoch took 521.837245 s
+ 
+2022-05-19 23:13:57.556623: 
+epoch:  78 
+2022-05-19 23:21:48.694082: train loss : 0.1020 
+2022-05-19 23:22:15.095253: validation loss: 0.1360 
+2022-05-19 23:22:15.095980: Average global foreground Dice: [0.6268, 0.8481] 
+2022-05-19 23:22:15.096067: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 23:22:15.508937: lr: 0.000104 
+2022-05-19 23:22:15.509162: saving scheduled checkpoint file... 
+2022-05-19 23:22:15.562340: saving checkpoint... 
+2022-05-19 23:22:16.491808: done, saving took 0.98 seconds 
+2022-05-19 23:22:16.501440: done 
+2022-05-19 23:23:46.252426: This epoch took 588.695693 s
+ 
+2022-05-19 23:23:46.252746: 
+epoch:  79 
+2022-05-19 23:31:37.754753: train loss : 0.1019 
+2022-05-19 23:32:04.079164: validation loss: 0.1164 
+2022-05-19 23:32:04.079794: Average global foreground Dice: [0.6178, 0.858] 
+2022-05-19 23:32:04.079901: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 23:32:04.494575: lr: 0.0001 
+2022-05-19 23:32:04.494790: saving scheduled checkpoint file... 
+2022-05-19 23:32:04.547216: saving checkpoint... 
+2022-05-19 23:32:05.491689: done, saving took 1.00 seconds 
+2022-05-19 23:32:05.501075: done 
+2022-05-19 23:32:26.758811: This epoch took 520.505951 s
+ 
+2022-05-19 23:32:26.759151: 
+epoch:  80 
+2022-05-19 23:40:18.601043: train loss : 0.1096 
+2022-05-19 23:40:44.935995: validation loss: 0.1081 
+2022-05-19 23:40:44.936596: Average global foreground Dice: [0.6288, 0.8453] 
+2022-05-19 23:40:44.936678: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 23:40:45.349712: lr: 9.6e-05 
+2022-05-19 23:40:45.349933: saving scheduled checkpoint file... 
+2022-05-19 23:40:45.401574: saving checkpoint... 
+2022-05-19 23:40:46.374263: done, saving took 1.02 seconds 
+2022-05-19 23:40:46.384009: done 
+2022-05-19 23:41:04.100768: This epoch took 517.341507 s
+ 
+2022-05-19 23:41:04.101078: 
+epoch:  81 
+2022-05-19 23:48:55.173277: train loss : 0.1031 
+2022-05-19 23:49:21.487772: validation loss: 0.1328 
+2022-05-19 23:49:21.488401: Average global foreground Dice: [0.6338, 0.8411] 
+2022-05-19 23:49:21.488488: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 23:49:21.896226: lr: 9.2e-05 
+2022-05-19 23:49:21.896455: saving scheduled checkpoint file... 
+2022-05-19 23:49:21.948079: saving checkpoint... 
+2022-05-19 23:49:22.787215: done, saving took 0.89 seconds 
+2022-05-19 23:49:22.795243: done 
+2022-05-19 23:49:53.547502: This epoch took 529.446245 s
+ 
+2022-05-19 23:49:53.547791: 
+epoch:  82 
+2022-05-19 23:57:45.126702: train loss : 0.1039 
+2022-05-19 23:58:11.455153: validation loss: 0.1031 
+2022-05-19 23:58:11.455822: Average global foreground Dice: [0.6542, 0.8079] 
+2022-05-19 23:58:11.455908: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-19 23:58:11.869607: lr: 8.8e-05 
+2022-05-19 23:58:11.869792: saving scheduled checkpoint file... 
+2022-05-19 23:58:11.920667: saving checkpoint... 
+2022-05-19 23:58:12.876374: done, saving took 1.01 seconds 
+2022-05-19 23:58:12.885792: done 
+2022-05-19 23:58:36.540402: This epoch took 522.992498 s
+ 
+2022-05-19 23:58:36.540703: 
+epoch:  83 
+2022-05-20 00:06:28.234009: train loss : 0.1017 
+2022-05-20 00:06:54.558066: validation loss: 0.1203 
+2022-05-20 00:06:54.558773: Average global foreground Dice: [0.6339, 0.8592] 
+2022-05-20 00:06:54.558858: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:06:54.966769: lr: 8.5e-05 
+2022-05-20 00:06:54.966974: saving scheduled checkpoint file... 
+2022-05-20 00:06:55.019888: saving checkpoint... 
+2022-05-20 00:06:55.936837: done, saving took 0.97 seconds 
+2022-05-20 00:06:55.946531: done 
+2022-05-20 00:07:15.500197: This epoch took 518.959382 s
+ 
+2022-05-20 00:07:15.500503: 
+epoch:  84 
+2022-05-20 00:15:07.352095: train loss : 0.1042 
+2022-05-20 00:15:33.680643: validation loss: 0.1308 
+2022-05-20 00:15:33.681300: Average global foreground Dice: [0.6427, 0.8537] 
+2022-05-20 00:15:33.681387: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:15:34.084127: lr: 8.1e-05 
+2022-05-20 00:15:34.084335: saving scheduled checkpoint file... 
+2022-05-20 00:15:34.136832: saving checkpoint... 
+2022-05-20 00:15:35.066612: done, saving took 0.98 seconds 
+2022-05-20 00:15:35.074948: done 
+2022-05-20 00:17:03.181084: This epoch took 587.680459 s
+ 
+2022-05-20 00:17:03.181515: 
+epoch:  85 
+2022-05-20 00:24:54.542538: train loss : 0.1049 
+2022-05-20 00:25:20.865211: validation loss: 0.1240 
+2022-05-20 00:25:20.865885: Average global foreground Dice: [0.6303, 0.8546] 
+2022-05-20 00:25:20.866011: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:25:21.265488: lr: 7.7e-05 
+2022-05-20 00:25:21.265695: saving scheduled checkpoint file... 
+2022-05-20 00:25:21.317370: saving checkpoint... 
+2022-05-20 00:25:22.143121: done, saving took 0.88 seconds 
+2022-05-20 00:25:22.151938: done 
+2022-05-20 00:28:43.128512: This epoch took 699.946877 s
+ 
+2022-05-20 00:28:43.128819: 
+epoch:  86 
+2022-05-20 00:36:34.436306: train loss : 0.1015 
+2022-05-20 00:37:00.770818: validation loss: 0.1247 
+2022-05-20 00:37:00.771546: Average global foreground Dice: [0.613, 0.8285] 
+2022-05-20 00:37:00.771630: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:37:01.170964: lr: 7.4e-05 
+2022-05-20 00:37:01.171159: saving scheduled checkpoint file... 
+2022-05-20 00:37:01.223203: saving checkpoint... 
+2022-05-20 00:37:02.199708: done, saving took 1.03 seconds 
+2022-05-20 00:37:02.209254: done 
+2022-05-20 00:37:26.599241: This epoch took 523.470312 s
+ 
+2022-05-20 00:37:26.599560: 
+epoch:  87 
+2022-05-20 00:45:17.708436: train loss : 0.0980 
+2022-05-20 00:45:44.049387: validation loss: 0.1210 
+2022-05-20 00:45:44.049983: Average global foreground Dice: [0.6293, 0.8585] 
+2022-05-20 00:45:44.050079: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:45:44.462501: lr: 7e-05 
+2022-05-20 00:45:44.462708: saving scheduled checkpoint file... 
+2022-05-20 00:45:44.515059: saving checkpoint... 
+2022-05-20 00:45:45.328977: done, saving took 0.87 seconds 
+2022-05-20 00:45:45.337806: done 
+2022-05-20 00:46:03.925183: This epoch took 517.325508 s
+ 
+2022-05-20 00:46:03.925475: 
+epoch:  88 
+2022-05-20 00:53:55.753352: train loss : 0.1008 
+2022-05-20 00:54:22.117807: validation loss: 0.1083 
+2022-05-20 00:54:22.118433: Average global foreground Dice: [0.619, 0.8445] 
+2022-05-20 00:54:22.118520: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 00:54:22.522735: lr: 6.7e-05 
+2022-05-20 00:54:22.522942: saving scheduled checkpoint file... 
+2022-05-20 00:54:22.575411: saving checkpoint... 
+2022-05-20 00:54:23.507722: done, saving took 0.98 seconds 
+2022-05-20 00:54:23.517888: done 
+2022-05-20 00:54:41.384289: This epoch took 517.458702 s
+ 
+2022-05-20 00:54:41.384619: 
+epoch:  89 
+2022-05-20 01:02:33.513410: train loss : 0.0984 
+2022-05-20 01:02:59.893457: validation loss: 0.1084 
+2022-05-20 01:02:59.894064: Average global foreground Dice: [0.6251, 0.8562] 
+2022-05-20 01:02:59.894159: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:03:00.295628: lr: 6.3e-05 
+2022-05-20 01:03:00.295824: saving scheduled checkpoint file... 
+2022-05-20 01:03:00.347775: saving checkpoint... 
+2022-05-20 01:03:01.273407: done, saving took 0.98 seconds 
+2022-05-20 01:03:01.283113: done 
+2022-05-20 01:03:18.312431: This epoch took 516.927703 s
+ 
+2022-05-20 01:03:18.312745: 
+epoch:  90 
+2022-05-20 01:11:10.211580: train loss : 0.0982 
+2022-05-20 01:11:36.570696: validation loss: 0.1058 
+2022-05-20 01:11:36.571369: Average global foreground Dice: [0.6349, 0.8587] 
+2022-05-20 01:11:36.571456: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:11:36.977980: lr: 6e-05 
+2022-05-20 01:11:36.978197: saving scheduled checkpoint file... 
+2022-05-20 01:11:37.031275: saving checkpoint... 
+2022-05-20 01:11:37.827788: done, saving took 0.85 seconds 
+2022-05-20 01:11:37.835813: done 
+2022-05-20 01:13:26.951646: This epoch took 608.638784 s
+ 
+2022-05-20 01:13:26.951973: 
+epoch:  91 
+2022-05-20 01:21:19.469252: train loss : 0.0958 
+2022-05-20 01:21:45.811795: validation loss: 0.1163 
+2022-05-20 01:21:45.812330: Average global foreground Dice: [0.6339, 0.856] 
+2022-05-20 01:21:45.812414: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:21:46.213187: lr: 5.7e-05 
+2022-05-20 01:21:46.213390: saving scheduled checkpoint file... 
+2022-05-20 01:21:46.265514: saving checkpoint... 
+2022-05-20 01:21:47.122302: done, saving took 0.91 seconds 
+2022-05-20 01:21:47.130528: done 
+2022-05-20 01:22:12.649498: This epoch took 525.697405 s
+ 
+2022-05-20 01:22:12.649810: 
+epoch:  92 
+2022-05-20 01:30:04.032043: train loss : 0.1002 
+2022-05-20 01:30:30.354064: validation loss: 0.1185 
+2022-05-20 01:30:30.354711: Average global foreground Dice: [0.6387, 0.8538] 
+2022-05-20 01:30:30.354801: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:30:30.756380: lr: 5.4e-05 
+2022-05-20 01:30:30.756582: saving scheduled checkpoint file... 
+2022-05-20 01:30:30.808923: saving checkpoint... 
+2022-05-20 01:30:31.793130: done, saving took 1.04 seconds 
+2022-05-20 01:30:31.802863: done 
+2022-05-20 01:31:01.605214: This epoch took 528.955278 s
+ 
+2022-05-20 01:31:01.605654: 
+epoch:  93 
+2022-05-20 01:38:53.267040: train loss : 0.0967 
+2022-05-20 01:39:19.595411: validation loss: 0.1152 
+2022-05-20 01:39:19.596012: Average global foreground Dice: [0.6315, 0.8593] 
+2022-05-20 01:39:19.596096: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:39:19.997767: lr: 5.1e-05 
+2022-05-20 01:39:19.997963: saving scheduled checkpoint file... 
+2022-05-20 01:39:20.050242: saving checkpoint... 
+2022-05-20 01:39:20.886524: done, saving took 0.89 seconds 
+2022-05-20 01:39:20.894508: done 
+2022-05-20 01:42:28.062499: This epoch took 686.456728 s
+ 
+2022-05-20 01:42:28.062840: 
+epoch:  94 
+2022-05-20 01:50:20.017456: train loss : 0.1003 
+2022-05-20 01:50:46.362335: validation loss: 0.1336 
+2022-05-20 01:50:46.362880: Average global foreground Dice: [0.6105, 0.8429] 
+2022-05-20 01:50:46.363023: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:50:46.767658: lr: 4.8e-05 
+2022-05-20 01:50:46.767870: saving scheduled checkpoint file... 
+2022-05-20 01:50:46.820096: saving checkpoint... 
+2022-05-20 01:50:47.736527: done, saving took 0.97 seconds 
+2022-05-20 01:50:47.746044: done 
+2022-05-20 01:51:08.281750: This epoch took 520.218798 s
+ 
+2022-05-20 01:51:08.282052: 
+epoch:  95 
+2022-05-20 01:58:59.877185: train loss : 0.1001 
+2022-05-20 01:59:26.236571: validation loss: 0.1172 
+2022-05-20 01:59:26.237278: Average global foreground Dice: [0.6491, 0.8619] 
+2022-05-20 01:59:26.237365: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 01:59:26.636260: lr: 4.5e-05 
+2022-05-20 01:59:26.636457: saving scheduled checkpoint file... 
+2022-05-20 01:59:26.687738: saving checkpoint... 
+2022-05-20 01:59:27.466919: done, saving took 0.83 seconds 
+2022-05-20 01:59:27.475549: done 
+2022-05-20 01:59:53.023152: This epoch took 524.740975 s
+ 
+2022-05-20 01:59:53.023460: 
+epoch:  96 
+2022-05-20 02:07:44.268248: train loss : 0.0998 
+2022-05-20 02:08:10.591882: validation loss: 0.1221 
+2022-05-20 02:08:10.592546: Average global foreground Dice: [0.631, 0.8314] 
+2022-05-20 02:08:10.592634: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:08:10.994893: lr: 4.2e-05 
+2022-05-20 02:08:10.995090: saving scheduled checkpoint file... 
+2022-05-20 02:08:11.047369: saving checkpoint... 
+2022-05-20 02:08:11.965285: done, saving took 0.97 seconds 
+2022-05-20 02:08:11.974049: done 
+2022-05-20 02:08:43.907636: This epoch took 530.884061 s
+ 
+2022-05-20 02:08:43.907938: 
+epoch:  97 
+2022-05-20 02:16:35.550293: train loss : 0.0944 
+2022-05-20 02:17:01.880734: validation loss: 0.1243 
+2022-05-20 02:17:01.881325: Average global foreground Dice: [0.6381, 0.8535] 
+2022-05-20 02:17:01.881420: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:17:02.286358: lr: 3.9e-05 
+2022-05-20 02:17:02.286548: saving scheduled checkpoint file... 
+2022-05-20 02:17:02.338381: saving checkpoint... 
+2022-05-20 02:17:03.328708: done, saving took 1.04 seconds 
+2022-05-20 02:17:03.337584: done 
+2022-05-20 02:17:29.167714: This epoch took 525.259665 s
+ 
+2022-05-20 02:17:29.168004: 
+epoch:  98 
+2022-05-20 02:25:21.013715: train loss : 0.0988 
+2022-05-20 02:25:47.364299: validation loss: 0.1448 
+2022-05-20 02:25:47.365052: Average global foreground Dice: [0.6265, 0.8502] 
+2022-05-20 02:25:47.365150: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:25:47.767186: lr: 3.6e-05 
+2022-05-20 02:25:47.767388: saving scheduled checkpoint file... 
+2022-05-20 02:25:47.819592: saving checkpoint... 
+2022-05-20 02:25:48.755710: done, saving took 0.99 seconds 
+2022-05-20 02:25:48.765253: done 
+2022-05-20 02:28:07.864649: This epoch took 638.696529 s
+ 
+2022-05-20 02:28:07.864943: 
+epoch:  99 
+2022-05-20 02:35:59.198253: train loss : 0.0939 
+2022-05-20 02:36:25.520954: validation loss: 0.1165 
+2022-05-20 02:36:25.521577: Average global foreground Dice: [0.6436, 0.8525] 
+2022-05-20 02:36:25.521663: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:36:25.927535: lr: 3.4e-05 
+2022-05-20 02:36:25.927735: saving scheduled checkpoint file... 
+2022-05-20 02:36:25.980280: saving checkpoint... 
+2022-05-20 02:36:26.921046: done, saving took 0.99 seconds 
+2022-05-20 02:36:26.930815: done 
+2022-05-20 02:36:54.334535: This epoch took 526.469486 s
+ 
+2022-05-20 02:36:54.334835: 
+epoch:  100 
+2022-05-20 02:44:45.652627: train loss : 0.0979 
+2022-05-20 02:45:11.994721: validation loss: 0.1243 
+2022-05-20 02:45:11.995292: Average global foreground Dice: [0.6365, 0.8596] 
+2022-05-20 02:45:11.995375: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:45:12.401132: lr: 3.1e-05 
+2022-05-20 02:45:12.401325: saving scheduled checkpoint file... 
+2022-05-20 02:45:12.453279: saving checkpoint... 
+2022-05-20 02:45:13.373629: done, saving took 0.97 seconds 
+2022-05-20 02:45:13.382883: done 
+2022-05-20 02:45:39.618601: This epoch took 525.283652 s
+ 
+2022-05-20 02:45:39.618912: 
+epoch:  101 
+2022-05-20 02:53:31.211427: train loss : 0.0937 
+2022-05-20 02:53:57.576347: validation loss: 0.1245 
+2022-05-20 02:53:57.576890: Average global foreground Dice: [0.6152, 0.8626] 
+2022-05-20 02:53:57.576975: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 02:53:57.984055: lr: 2.9e-05 
+2022-05-20 02:53:57.984264: saving scheduled checkpoint file... 
+2022-05-20 02:53:58.036330: saving checkpoint... 
+2022-05-20 02:53:58.962407: done, saving took 0.98 seconds 
+2022-05-20 02:53:58.972052: done 
+2022-05-20 02:54:20.832186: This epoch took 521.213156 s
+ 
+2022-05-20 02:54:20.832523: 
+epoch:  102 
+2022-05-20 03:02:12.280318: train loss : 0.0934 
+2022-05-20 03:02:38.666991: validation loss: 0.1375 
+2022-05-20 03:02:38.667647: Average global foreground Dice: [0.6343, 0.8601] 
+2022-05-20 03:02:38.667732: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:02:39.070033: lr: 2.6e-05 
+2022-05-20 03:02:39.070246: saving scheduled checkpoint file... 
+2022-05-20 03:02:39.122334: saving checkpoint... 
+2022-05-20 03:02:40.129189: done, saving took 1.06 seconds 
+2022-05-20 03:02:40.137928: done 
+2022-05-20 03:05:48.942263: This epoch took 688.109630 s
+ 
+2022-05-20 03:05:48.942561: 
+epoch:  103 
+2022-05-20 03:13:41.032162: train loss : 0.1003 
+2022-05-20 03:14:07.360496: validation loss: 0.1131 
+2022-05-20 03:14:07.361150: Average global foreground Dice: [0.6326, 0.8471] 
+2022-05-20 03:14:07.361236: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:14:07.767490: lr: 2.4e-05 
+2022-05-20 03:14:07.767702: saving scheduled checkpoint file... 
+2022-05-20 03:14:07.820111: saving checkpoint... 
+2022-05-20 03:14:08.638200: done, saving took 0.87 seconds 
+2022-05-20 03:14:08.646991: done 
+2022-05-20 03:14:40.265126: This epoch took 531.322451 s
+ 
+2022-05-20 03:14:40.265419: 
+epoch:  104 
+2022-05-20 03:22:32.252051: train loss : 0.1000 
+2022-05-20 03:22:58.600253: validation loss: 0.1231 
+2022-05-20 03:22:58.600855: Average global foreground Dice: [0.6363, 0.8609] 
+2022-05-20 03:22:58.600941: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:22:59.004965: lr: 2.2e-05 
+2022-05-20 03:22:59.005167: saving scheduled checkpoint file... 
+2022-05-20 03:22:59.057498: saving checkpoint... 
+2022-05-20 03:22:59.996806: done, saving took 0.99 seconds 
+2022-05-20 03:23:00.006322: done 
+2022-05-20 03:26:17.375167: This epoch took 697.109617 s
+ 
+2022-05-20 03:26:17.375500: 
+epoch:  105 
+2022-05-20 03:34:09.052386: train loss : 0.0949 
+2022-05-20 03:34:35.405204: validation loss: 0.1240 
+2022-05-20 03:34:35.405837: Average global foreground Dice: [0.6289, 0.8445] 
+2022-05-20 03:34:35.405925: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:34:35.808467: lr: 2e-05 
+2022-05-20 03:34:35.808662: saving scheduled checkpoint file... 
+2022-05-20 03:34:35.860185: saving checkpoint... 
+2022-05-20 03:34:36.782282: done, saving took 0.97 seconds 
+2022-05-20 03:34:36.791335: done 
+2022-05-20 03:37:41.530821: This epoch took 684.155207 s
+ 
+2022-05-20 03:37:41.531134: 
+epoch:  106 
+2022-05-20 03:45:33.236562: train loss : 0.0954 
+2022-05-20 03:45:59.574929: validation loss: 0.1532 
+2022-05-20 03:45:59.575588: Average global foreground Dice: [0.6228, 0.8469] 
+2022-05-20 03:45:59.575673: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:45:59.995384: lr: 1.8e-05 
+2022-05-20 03:45:59.995584: saving scheduled checkpoint file... 
+2022-05-20 03:46:00.047075: saving checkpoint... 
+2022-05-20 03:46:00.978893: done, saving took 0.98 seconds 
+2022-05-20 03:46:00.988956: done 
+2022-05-20 03:46:23.181058: This epoch took 521.649814 s
+ 
+2022-05-20 03:46:23.181443: 
+epoch:  107 
+2022-05-20 03:54:14.554516: train loss : 0.0993 
+2022-05-20 03:54:40.881835: validation loss: 0.1141 
+2022-05-20 03:54:40.882494: Average global foreground Dice: [0.6254, 0.8507] 
+2022-05-20 03:54:40.882581: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 03:54:41.286762: lr: 1.6e-05 
+2022-05-20 03:54:41.286955: saving scheduled checkpoint file... 
+2022-05-20 03:54:41.338754: saving checkpoint... 
+2022-05-20 03:54:42.326008: done, saving took 1.04 seconds 
+2022-05-20 03:54:42.335795: done 
+2022-05-20 03:56:40.975352: This epoch took 617.793770 s
+ 
+2022-05-20 03:56:40.975672: 
+epoch:  108 
+2022-05-20 04:04:32.644532: train loss : 0.1000 
+2022-05-20 04:04:58.969759: validation loss: 0.1226 
+2022-05-20 04:04:58.970464: Average global foreground Dice: [0.6307, 0.851] 
+2022-05-20 04:04:58.970550: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 04:04:59.377042: lr: 1.4e-05 
+2022-05-20 04:04:59.377254: saving scheduled checkpoint file... 
+2022-05-20 04:04:59.429801: saving checkpoint... 
+2022-05-20 04:05:00.355347: done, saving took 0.98 seconds 
+2022-05-20 04:05:00.365554: done 
+2022-05-20 04:05:22.335599: This epoch took 521.359818 s
+ 
+2022-05-20 04:05:22.335909: 
+epoch:  109 
+2022-05-20 04:13:13.812703: train loss : 0.0915 
+2022-05-20 04:13:40.113782: validation loss: 0.1293 
+2022-05-20 04:13:40.114422: Average global foreground Dice: [0.6202, 0.8523] 
+2022-05-20 04:13:40.114508: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 04:13:40.530390: lr: 1.2e-05 
+2022-05-20 04:13:40.530597: saving scheduled checkpoint file... 
+2022-05-20 04:13:40.582570: saving checkpoint... 
+2022-05-20 04:13:41.503951: done, saving took 0.97 seconds 
+2022-05-20 04:13:41.512445: done 
+2022-05-20 04:14:03.764734: This epoch took 521.428715 s
+ 
+2022-05-20 04:14:03.765044: 
+epoch:  110 
+2022-05-20 04:21:55.725989: train loss : 0.0942 
+2022-05-20 04:22:22.036795: validation loss: 0.1194 
+2022-05-20 04:22:22.037431: Average global foreground Dice: [0.6314, 0.8651] 
+2022-05-20 04:22:22.037520: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 04:22:22.452833: lr: 1.1e-05 
+2022-05-20 04:22:22.453032: saving scheduled checkpoint file... 
+2022-05-20 04:22:22.505664: saving checkpoint... 
+2022-05-20 04:22:23.434414: done, saving took 0.98 seconds 
+2022-05-20 04:22:23.444342: done 
+2022-05-20 04:22:44.076867: This epoch took 520.311683 s
+ 
+2022-05-20 04:22:44.077249: 
+epoch:  111 
+2022-05-20 04:30:35.312382: train loss : 0.0924 
+2022-05-20 04:31:01.651766: validation loss: 0.1230 
+2022-05-20 04:31:01.652409: Average global foreground Dice: [0.6343, 0.8449] 
+2022-05-20 04:31:01.652494: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 04:31:02.057842: lr: 9e-06 
+2022-05-20 04:31:02.058036: saving scheduled checkpoint file... 
+2022-05-20 04:31:02.110332: saving checkpoint... 
+2022-05-20 04:31:03.050282: done, saving took 0.99 seconds 
+2022-05-20 04:31:03.060034: done 
+2022-05-20 04:32:52.310267: This epoch took 608.232898 s
+ 
+2022-05-20 04:32:52.310611: 
+epoch:  112 
+2022-05-20 04:40:43.789970: train loss : 0.0894 
+2022-05-20 04:41:10.128337: validation loss: 0.1268 
+2022-05-20 04:41:10.129044: Average global foreground Dice: [0.6437, 0.8632] 
+2022-05-20 04:41:10.129141: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 04:41:10.530688: lr: 8e-06 
+2022-05-20 04:41:10.530893: saving scheduled checkpoint file... 
+2022-05-20 04:41:10.583108: saving checkpoint... 
+2022-05-20 04:41:11.519285: done, saving took 0.99 seconds 
+2022-05-20 04:41:11.529615: done 
+2022-05-20 04:41:29.566789: This epoch took 517.256067 s
+ 
+2022-05-20 04:41:29.567090: 
+epoch:  113 
+2022-05-20 04:49:20.947436: train loss : 0.0966 
+2022-05-20 04:49:47.284910: validation loss: 0.1468 
+2022-05-20 04:49:47.285512: Average global foreground Dice: [0.618, 0.8502] 
+2022-05-20 04:49:47.285598: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 04:49:47.691832: lr: 7e-06 
+2022-05-20 04:49:47.692035: saving scheduled checkpoint file... 
+2022-05-20 04:49:47.744670: saving checkpoint... 
+2022-05-20 04:49:48.684906: done, saving took 0.99 seconds 
+2022-05-20 04:49:48.694838: done 
+2022-05-20 04:50:09.197347: This epoch took 519.630142 s
+ 
+2022-05-20 04:50:09.197633: 
+epoch:  114 
+2022-05-20 04:58:01.186598: train loss : 0.0925 
+2022-05-20 04:58:27.537900: validation loss: 0.1089 
+2022-05-20 04:58:27.538580: Average global foreground Dice: [0.6332, 0.8375] 
+2022-05-20 04:58:27.538666: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 04:58:27.943815: lr: 6e-06 
+2022-05-20 04:58:27.944010: saving scheduled checkpoint file... 
+2022-05-20 04:58:27.995519: saving checkpoint... 
+2022-05-20 04:58:28.926910: done, saving took 0.98 seconds 
+2022-05-20 04:58:28.936626: done 
+2022-05-20 05:01:42.696367: This epoch took 693.498621 s
+ 
+2022-05-20 05:01:42.696679: 
+epoch:  115 
+2022-05-20 05:09:34.346075: train loss : 0.0918 
+2022-05-20 05:10:00.666482: validation loss: 0.1287 
+2022-05-20 05:10:00.667158: Average global foreground Dice: [0.6204, 0.8482] 
+2022-05-20 05:10:00.667245: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 05:10:01.072694: lr: 5e-06 
+2022-05-20 05:10:01.072886: saving scheduled checkpoint file... 
+2022-05-20 05:10:01.124359: saving checkpoint... 
+2022-05-20 05:10:02.036735: done, saving took 0.96 seconds 
+2022-05-20 05:10:02.046803: done 
+2022-05-20 05:10:26.429659: This epoch took 523.732872 s
+ 
+2022-05-20 05:10:26.429962: 
+epoch:  116 
+2022-05-20 05:18:17.464725: train loss : 0.0892 
+2022-05-20 05:18:43.790609: validation loss: 0.1289 
+2022-05-20 05:18:43.791240: Average global foreground Dice: [0.6553, 0.869] 
+2022-05-20 05:18:43.791324: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 05:18:44.193709: lr: 4e-06 
+2022-05-20 05:18:44.193909: saving scheduled checkpoint file... 
+2022-05-20 05:18:44.245325: saving checkpoint... 
+2022-05-20 05:18:45.182998: done, saving took 0.99 seconds 
+2022-05-20 05:18:45.192671: done 
+2022-05-20 05:19:03.726619: This epoch took 517.296544 s
+ 
+2022-05-20 05:19:03.726914: 
+epoch:  117 
+2022-05-20 05:26:55.115848: train loss : 0.1028 
+2022-05-20 05:27:21.442067: validation loss: 0.1189 
+2022-05-20 05:27:21.442729: Average global foreground Dice: [0.6368, 0.8535] 
+2022-05-20 05:27:21.442820: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 05:27:21.853065: lr: 3e-06 
+2022-05-20 05:27:21.853264: saving scheduled checkpoint file... 
+2022-05-20 05:27:21.905330: saving checkpoint... 
+2022-05-20 05:27:22.731925: done, saving took 0.88 seconds 
+2022-05-20 05:27:22.740766: done 
+2022-05-20 05:27:43.240691: This epoch took 519.513662 s
+ 
+2022-05-20 05:27:43.241021: 
+epoch:  118 
+2022-05-20 05:35:34.916461: train loss : 0.0958 
+2022-05-20 05:36:01.299965: validation loss: 0.1195 
+2022-05-20 05:36:01.300652: Average global foreground Dice: [0.6323, 0.8559] 
+2022-05-20 05:36:01.300737: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 05:36:01.706110: lr: 2e-06 
+2022-05-20 05:36:01.706300: saving scheduled checkpoint file... 
+2022-05-20 05:36:01.757981: saving checkpoint... 
+2022-05-20 05:36:02.699279: done, saving took 0.99 seconds 
+2022-05-20 05:36:02.708307: done 
+2022-05-20 05:36:22.913836: This epoch took 519.672703 s
+ 
+2022-05-20 05:36:22.914159: 
+epoch:  119 
+2022-05-20 05:44:14.476660: train loss : 0.0903 
+2022-05-20 05:44:40.818745: validation loss: 0.1210 
+2022-05-20 05:44:40.819325: Average global foreground Dice: [0.6312, 0.843] 
+2022-05-20 05:44:40.819409: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 05:44:41.226760: lr: 1e-06 
+2022-05-20 05:44:41.226958: saving scheduled checkpoint file... 
+2022-05-20 05:44:41.279076: saving checkpoint... 
+2022-05-20 05:44:42.207369: done, saving took 0.98 seconds 
+2022-05-20 05:44:42.216247: done 
+2022-05-20 05:46:34.731322: This epoch took 611.817053 s
+ 
+2022-05-20 05:46:34.731674: 
+epoch:  120 
+2022-05-20 05:54:26.176627: train loss : 0.0946 
+2022-05-20 05:54:52.549079: validation loss: 0.1347 
+2022-05-20 05:54:52.549742: Average global foreground Dice: [0.6324, 0.8419] 
+2022-05-20 05:54:52.549827: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 05:54:52.957762: lr: 1e-06 
+2022-05-20 05:54:52.957964: saving scheduled checkpoint file... 
+2022-05-20 05:54:53.009961: saving checkpoint... 
+2022-05-20 05:54:53.928981: done, saving took 0.97 seconds 
+2022-05-20 05:54:53.938824: done 
+2022-05-20 05:55:16.334688: This epoch took 521.602904 s
+ 
+2022-05-20 05:55:16.335009: 
+epoch:  121 
+2022-05-20 06:03:07.710399: train loss : 0.0942 
+2022-05-20 06:03:34.044385: validation loss: 0.1102 
+2022-05-20 06:03:34.044974: Average global foreground Dice: [0.6333, 0.846] 
+2022-05-20 06:03:34.045056: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 06:03:34.450281: lr: 1e-06 
+2022-05-20 06:03:34.450484: saving scheduled checkpoint file... 
+2022-05-20 06:03:34.502645: saving checkpoint... 
+2022-05-20 06:03:35.344863: done, saving took 0.89 seconds 
+2022-05-20 06:03:35.353413: done 
+2022-05-20 06:04:27.428057: This epoch took 551.092916 s
+ 
+2022-05-20 06:04:27.428370: 
+epoch:  122 
+2022-05-20 06:12:18.864177: train loss : 0.0996 
+2022-05-20 06:12:45.195073: validation loss: 0.1357 
+2022-05-20 06:12:45.195686: Average global foreground Dice: [0.6336, 0.8554] 
+2022-05-20 06:12:45.195814: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 06:12:45.604357: lr: 0.0 
+2022-05-20 06:12:45.604557: saving scheduled checkpoint file... 
+2022-05-20 06:12:45.657175: saving checkpoint... 
+2022-05-20 06:12:46.580700: done, saving took 0.98 seconds 
+2022-05-20 06:12:46.590537: done 
+2022-05-20 06:15:12.863706: This epoch took 645.435229 s
+ 
+2022-05-20 06:15:12.864010: 
+epoch:  123 
+2022-05-20 06:23:04.519852: train loss : 0.0890 
+2022-05-20 06:23:30.856823: validation loss: 0.1017 
+2022-05-20 06:23:30.857710: Average global foreground Dice: [0.633, 0.8658] 
+2022-05-20 06:23:30.857796: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 06:23:31.264113: lr: 0.0 
+2022-05-20 06:23:31.264309: saving scheduled checkpoint file... 
+2022-05-20 06:23:31.316732: saving checkpoint... 
+2022-05-20 06:23:32.185140: done, saving took 0.92 seconds 
+2022-05-20 06:23:32.207525: done 
+2022-05-20 06:23:49.408279: This epoch took 516.544146 s
+ 
+2022-05-20 06:23:49.408664: 
+epoch:  124 
+2022-05-20 06:31:40.682584: train loss : 0.0959 
+2022-05-20 06:32:07.023501: validation loss: 0.1012 
+2022-05-20 06:32:07.024133: Average global foreground Dice: [0.6263, 0.8546] 
+2022-05-20 06:32:07.024221: (interpret this as an estimate for the Dice of the different classes. This is not exact.) 
+2022-05-20 06:32:07.431305: lr: 0.0 
+2022-05-20 06:32:07.431503: saving scheduled checkpoint file... 
+2022-05-20 06:32:07.483509: saving checkpoint... 
+2022-05-20 06:32:08.413666: done, saving took 0.98 seconds 
+2022-05-20 06:32:08.423411: done 
+2022-05-20 06:34:36.347142: This epoch took 646.938243 s
+ 
+2022-05-20 06:34:36.392874: saving checkpoint... 
+2022-05-20 06:34:37.017723: done, saving took 0.67 seconds 

Task008_HepaticVessel/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU008_64192192_max2former_ms-432_decl3_d192_bT-d1_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32/plans.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f86e3298caef295e5634424ec1e239a4b7b0f32613863d610242265817922ee8
+size 187584

Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/debug.json

@@ -0,0 +1,107 @@
+{
+    "all_tr_losses": "[]",
+    "all_val_eval_metrics": "[]",
+    "all_val_losses": "[]",
+    "all_val_losses_tr_mode": "[]",
+    "also_val_in_tr_mode": "False",
+    "amp_grad_scaler": "None",
+    "args": "Namespace(batch_size=2, config='./configs/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250.yaml', continue_training=False, crop_size=[128, 128, 128], dbs=False, deep_supervision_scales=[], deterministic=False, disable_decoder=False, disable_ds=True, disable_postprocessing_on_folds=False, disable_saving=False, drop_path=0.0, find_lr=False, find_zero_weight_decay=False, fix_ds_net_numpool=False, fold='0', fp32=False, hdfs_base='GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250', initial_lr=0.0003, is_sigmoid=False, is_spatial_aug_only=False, is_ssl=False, layer_decay=1.0, local_rank=0, loss_name='', lrschedule='warmup_cosine', mask_ratio=0.75, max_loss_cal='v1', max_num_epochs=250, merge_femur=False, min_lr=0, model='Generic_TransUNet_max_ppbp', model_params={'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': False, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}, n_class=17, network='3d_fullres', network_trainer='nnUNetTrainerV2_DDP', npz=False, opt_betas=None, opt_eps=1e-08, optim_name='adamw', p='nnUNetPlansv2.1', plan_update='', pretrained=False, pretrained_config=None, pretrained_weights=None, rank=0, reclip=None, resume='auto', skip_grad_nan=False, task='Task500_BraTS2021', task_pretrained='Task801_WORD', total_batch_size=None, use_compressed_data=False, val_final=False, val_folder='validation_raw', valbest=False, validation_only=False, vallatest=False, warmup_epochs=10, weight_decay=3e-05, world_size=8)",
+    "base_num_features": "32",
+    "basic_generator_patch_size": "[182 182 182]",
+    "batch_dice": "True",
+    "batch_size": "2",
+    "best_MA_tr_loss_for_patience": "None",
+    "best_epoch_based_on_MA_tr_loss": "None",
+    "best_val_eval_criterion_MA": "None",
+    "classes": "[1, 2, 3]",
+    "conv_per_stage": "2",
+    "data_aug_params": "{'selected_data_channels': None, 'selected_seg_channels': [0], 'do_elastic': True, 'elastic_deform_alpha': (0.0, 900.0), 'elastic_deform_sigma': (9.0, 13.0), 'p_eldef': 0.3, 'do_scaling': True, 'scale_range': (0.65, 1.6), 'independent_scale_factor_for_each_axis': True, 'p_independent_scale_per_axis': 0.3, 'p_scale': 0.3, 'do_rotation': True, 'rotation_x': (-1.5707963267948966, 1.5707963267948966), 'rotation_y': (-1.5707963267948966, 1.5707963267948966), 'rotation_z': (-1.5707963267948966, 1.5707963267948966), 'rotation_p_per_axis': 1, 'p_rot': 0.3, 'random_crop': False, 'random_crop_dist_to_border': None, 'do_gamma': True, 'gamma_retain_stats': True, 'gamma_range': (0.5, 1.6), 'p_gamma': 0.3, 'do_mirror': True, 'mirror_axes': (0, 1, 2), 'dummy_2D': False, 'mask_was_used_for_normalization': OrderedDict([(0, True), (1, True), (2, True), (3, True)]), 'border_mode_data': 'constant', 'all_segmentation_labels': None, 'move_last_seg_chanel_to_data': False, 'cascade_do_cascade_augmentations': False, 'cascade_random_binary_transform_p': 0.4, 'cascade_random_binary_transform_p_per_label': 1, 'cascade_random_binary_transform_size': (1, 8), 'cascade_remove_conn_comp_p': 0.2, 'cascade_remove_conn_comp_max_size_percent_threshold': 0.15, 'cascade_remove_conn_comp_fill_with_other_class_p': 0.0, 'do_additive_brightness': True, 'additive_brightness_p_per_sample': 0.3, 'additive_brightness_p_per_channel': 0.5, 'additive_brightness_mu': 0, 'additive_brightness_sigma': 0.2, 'num_threads': 12, 'num_cached_per_thread': 4, 'patch_size_for_spatialtransform': array([128, 128, 128]), 'eldef_deformation_scale': (0, 0.25)}",
+    "dataset_directory": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021",
+    "deep_supervision_scales": "[[1, 1, 1], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]]",
+    "deterministic": "False",
+    "disable_ds": "True",
+    "distribute_batch_size": "False",
+    "dl_tr": "<dataset_loading.DataLoader3D object at 0x7fab0eec9c88>",
+    "dl_val": "<dataset_loading.DataLoader3D object at 0x7fab28dd1ac8>",
+    "do_dummy_2D_aug": "False",
+    "ds_loss_weights": "[1. 0. 0. 0. 0.]",
+    "epoch": "0",
+    "experiment_name": "nnUNetTrainerV2_DDP",
+    "find_zero_weight_decay": "False",
+    "fold": "0",
+    "folder_with_preprocessed_data": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetData_plans_v2.1_stage0",
+    "fp16": "True",
+    "global_batch_size": "16",
+    "gt_niftis_folder": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/gt_segmentations",
+    "inference_pad_border_mode": "constant",
+    "inference_pad_kwargs": "{'constant_values': 0}",
+    "init_args": "('/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetPlansv2.1_plans_3D.pkl', 0, 0, '/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250', '/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021', True, 0, True, False, False, True)",
+    "initial_lr": "0.0003",
+    "input_size": "[128, 128, 128]",
+    "is_spatial_aug_only": "False",
+    "is_ssl": "False",
+    "layer_decay": "1.0",
+    "local_rank": "0",
+    "log_file": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/training_log_2022_5_14_04_20_21.txt",
+    "loss": "None",
+    "lr_scheduler": "<optimizers.lr_scheduler.LinearWarmupCosineAnnealingLR object at 0x7faabebdf400>",
+    "lr_scheduler_eps": "0.001",
+    "lr_scheduler_name": "warmup_cosine",
+    "lr_scheduler_patience": "30",
+    "lr_threshold": "1e-06",
+    "max_num_epochs": "250",
+    "min_lr": "0",
+    "min_region_size_per_class": "None",
+    "min_size_per_class": "None",
+    "model": "Generic_TransUNet_max_ppbp",
+    "model_params": "{'is_masked_attn': True, 'max_dec_layers': 3, 'is_max_bottleneck_transformer': False, 'max_msda': '', 'is_max_ms': True, 'max_ms_idxs': [-4, -3, -2], 'max_hidden_dim': 192, 'mw': 1.0, 'is_max_ds': True, 'is_masking': True, 'is_max_hungarian': True, 'num_queries': 20, 'is_max_cls': True, 'is_mhsa_float32': True}",
+    "net_conv_kernel_sizes": "[[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]]",
+    "net_num_pool_op_kernel_sizes": "[[2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2]]",
+    "net_pool_per_axis": "[5, 5, 5]",
+    "normalization_schemes": "OrderedDict([(0, 'nonCT'), (1, 'nonCT'), (2, 'nonCT'), (3, 'nonCT')])",
+    "num_batches_per_epoch": "250",
+    "num_classes": "3",
+    "num_input_channels": "4",
+    "num_val_batches_per_epoch": "50",
+    "online_eval_fn": "[]",
+    "online_eval_foreground_dc": "[]",
+    "online_eval_fp": "[]",
+    "online_eval_tp": "[]",
+    "only_keep_largest_connected_component": "None",
+    "optim_name": "adamw",
+    "optimizer": "AdamW (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    eps: 1e-08\n    initial_lr: 0.0003\n    lr: 0.0\n    weight_decay: 3e-05\n)",
+    "output_folder": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0",
+    "output_folder_base": "/opt/tiger/project/data/nnUNet_trained_models/UNet_IN_NANFang/Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250",
+    "oversample_foreground_percent": "0.0",
+    "pad_all_sides": "None",
+    "patch_size": "[128 128 128]",
+    "patience": "50",
+    "pin_memory": "True",
+    "plans_file": "/opt/tiger/project/data/nnUNet_preprocessed/Task500_BraTS2021/nnUNetPlansv2.1_plans_3D.pkl",
+    "reclip": "None",
+    "regions": "{'whole tumor': (1, 2, 3), 'tumor core': (2, 3), 'enhancing tumor': (3,)}",
+    "regions_class_order": "(1, 2, 3)",
+    "resume": "auto",
+    "save_best_checkpoint": "True",
+    "save_every": "1",
+    "save_final_checkpoint": "True",
+    "save_intermediate_checkpoints": "True",
+    "save_latest_only": "True",
+    "stage": "0",
+    "threeD": "True",
+    "tr_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7fab0eee2ac8>",
+    "train_loss_MA": "None",
+    "train_loss_MA_alpha": "0.93",
+    "train_loss_MA_eps": "0.0005",
+    "transpose_backward": "[0, 1, 2]",
+    "transpose_forward": "[0, 1, 2]",
+    "unpack_data": "True",
+    "use_mask_for_norm": "OrderedDict([(0, True), (1, True), (2, True), (3, True)])",
+    "use_progress_bar": "False",
+    "val_eval_criterion_MA": "None",
+    "val_eval_criterion_alpha": "0.9",
+    "val_gen": "<batchgenerators.dataloading.multi_threaded_augmenter.MultiThreadedAugmenter object at 0x7fab0f103828>",
+    "warmup_epochs": "10",
+    "was_initialized": "True",
+    "weight_decay": "3e-05"
+}

Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_best.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21f55c806780ce635c27d50b81a54164ad4e8892cafcdf660926f8ef2c5ce538
+size 409150945

Task500_BraTS2021/nnUNetTrainerV2_DDP__nnUNetPlansv2.1/GeTU500Region_128128128_max2former_ms-432_decl3_d192_mds_mw1.0_disds_bs2x8_adamw_warmup10_lr3e-4_masklossv1_masking_hungarian20_mhsa32_epo250/fold_0/model_best.model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecc67a3df1e496e8155d1d5cef774c62efcd0b6a34c6e6dadb6c7091fd12f1a2
+size 393843