Upload 15 files

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/config.yaml

@@ -0,0 +1,54 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 3
+mix_lst_path: ./data/VoxCeleb2/mixture_data_list_3mix.csv
+audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/
+reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/
+audio_sr: 16000
+ref_sr: 25
+
+# dataloader
+num_workers: 4
+batch_size: 2         # 4-GPU training with a total effective batch size of 8
+accu_grad: 0
+effec_batch_size: 2   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 3         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: None       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: lip            # lip or speech or gesture or EEG
+  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
+  emb_size: 256       # resnet18:256
+network_audio:
+  backbone: av_mossformer2
+  encoder_kernel_size: 16
+  encoder_out_nchannels: 512
+  encoder_in_nchannels: 1
+  
+  masknet_numspks: 1
+  masknet_chunksize: 250
+  masknet_numlayers: 1
+  masknet_norm: "ln"
+  masknet_useextralinearlayer: False
+  masknet_extraskipconnection: True
+
+  intra_numlayers: 24
+  intra_nhead: 8
+  intra_dffn: 1024
+  intra_dropout: 0
+  intra_use_positional: True
+  intra_norm_before: True
+        
+
+# optimizer
+loss_type: sisdr      # "snr", "sisdr", "hybrid"
+init_learning_rate: 0.00015
+max_epoch: 150
+clip_grad_norm: 5

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/last_best_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b43436d73ce61d73b3879db8d28a13c0ecbaef8ab35b576f7fb816bb51e3e82c
+size 734561014

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/last_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fdaf3c8f319bd29825df4211bd120b2b8dcf74ccbc02c0b0cfb345716380465
+size 734537584

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/log_2024-11-12(09:27:22).txt

@@ -0,0 +1,617 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 3
+mix_lst_path: ./data/VoxCeleb2/mixture_data_list_3mix.csv
+audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/
+reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/
+audio_sr: 16000
+ref_sr: 25
+
+# dataloader
+num_workers: 4
+batch_size: 2         # 2-GPU training with a total effective batch size of 8
+accu_grad: 1
+effec_batch_size: 4   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 3         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: None       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: lip            # lip or speech or gesture or EEG
+  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
+  emb_size: 256       # resnet18:256
+network_audio:
+  backbone: av_mossformer2
+  encoder_kernel_size: 16
+  encoder_out_nchannels: 512
+  encoder_in_nchannels: 1
+  
+  masknet_numspks: 1
+  masknet_chunksize: 250
+  masknet_numlayers: 1
+  masknet_norm: "ln"
+  masknet_useextralinearlayer: False
+  masknet_extraskipconnection: True
+
+  intra_numlayers: 24
+  intra_nhead: 8
+  intra_dffn: 1024
+  intra_dropout: 0
+  intra_use_positional: True
+  intra_norm_before: True
+        
+
+# optimizer
+loss_type: sisdr      # "snr", "sisdr", "hybrid"
+init_learning_rate: 0.00015
+max_epoch: 150
+clip_grad_norm: 5
+W1112 09:27:54.432088 139873319929664 torch/distributed/run.py:779] 
+W1112 09:27:54.432088 139873319929664 torch/distributed/run.py:779] *****************************************
+W1112 09:27:54.432088 139873319929664 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W1112 09:27:54.432088 139873319929664 torch/distributed/run.py:779] *****************************************
+started on checkpoints/log_2024-11-12(09:27:22)
+
+namespace(accu_grad=1, audio_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/audio_clean/', audio_sr=16000, batch_size=2, causal=0, checkpoint_dir='checkpoints/log_2024-11-12(09:27:22)', clip_grad_norm=5.0, config=[<yamlargparse.Path object at 0x7ff03ef79c10>], device=device(type='cuda'), distributed=True, effec_batch_size=4, evaluate_only=0, init_from='None', init_learning_rate=0.00015, local_rank=0, loss_type='sisdr', lr_warmup=0, max_epoch=150, max_length=3, mix_lst_path='./data/VoxCeleb2/mixture_data_list_3mix.csv', network_audio=namespace(backbone='av_mossformer2', encoder_in_nchannels=1, encoder_kernel_size=16, encoder_out_nchannels=512, intra_dffn=1024, intra_dropout=0, intra_nhead=8, intra_norm_before=True, intra_numlayers=24, intra_use_positional=True, masknet_chunksize=250, masknet_extraskipconnection=True, masknet_norm='ln', masknet_numlayers=1, masknet_numspks=1, masknet_useextralinearlayer=False), network_reference=namespace(backbone='resnet18', cue='lip', emb_size=256), num_workers=4, ref_sr=25, reference_direc='/mnt/nas_sg/wulanchabu/zexu.pan/datasets/VoxCeleb2/orig/', seed=777, speaker_no=3, train_from_last_checkpoint=0, use_cuda=1, world_size=2)
+network_wrapper(
+  (sep_network): av_Mossformer(
+    (encoder): Encoder(
+      (conv1d_U): Conv1d(1, 512, kernel_size=(16,), stride=(8,), bias=False)
+    )
+    (separator): Separator(
+      (layer_norm): GroupNorm(1, 512, eps=1e-08, affine=True)
+      (bottleneck_conv1x1): Conv1d(512, 512, kernel_size=(1,), stride=(1,), bias=False)
+      (masknet): Dual_Path_Model(
+        (pos_enc): ScaledSinuEmbedding()
+        (dual_mdl): ModuleList(
+          (0): Dual_Computation_Block(
+            (intra_mdl): SBFLASHBlock_DualA(
+              (mdl): TransformerEncoder_FLASH_DualA_FSMN(
+                (flashT): FLASHTransformer_DualA_FSMN(
+                  (fsmn): ModuleList(
+                    (0-23): 24 x Gated_FSMN_Block_Dilated(
+                      (conv1): Sequential(
+                        (0): Conv1d(512, 256, kernel_size=(1,), stride=(1,))
+                        (1): PReLU(num_parameters=1)
+                      )
+                      (norm1): CLayerNorm((256,), eps=1e-05, elementwise_affine=True)
+                      (gated_fsmn): Gated_FSMN_dilated(
+                        (to_u): FFConvM(
+                          (mdl): Sequential(
+                            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+                            (1): Linear(in_features=256, out_features=256, bias=True)
+                            (2): SiLU()
+                            (3): ConvModule(
+                              (sequential): Sequential(
+                                (0): Transpose()
+                                (1): DepthwiseConv1d(
+                                  (conv): Conv1d(256, 256, kernel_size=(17,), stride=(1,), padding=(8,), groups=256, bias=False)
+                                )
+                              )
+                            )
+                            (4): Dropout(p=0.1, inplace=False)
+                          )
+                        )
+                        (to_v): FFConvM(
+                          (mdl): Sequential(
+                            (0): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+                            (1): Linear(in_features=256, out_features=256, bias=True)
+                            (2): SiLU()
+                            (3): ConvModule(
+                              (sequential): Sequential(
+                                (0): Transpose()
+                                (1): DepthwiseConv1d(
+                                  (conv): Conv1d(256, 256, kernel_size=(17,), stride=(1,), padding=(8,), groups=256, bias=False)
+                                )
+                              )
+                            )
+                            (4): Dropout(p=0.1, inplace=False)
+                          )
+                        )
+                        (fsmn): UniDeepFsmn_dilated(
+                          (linear): Linear(in_features=256, out_features=256, bias=True)
+                          (project): Linear(in_features=256, out_features=256, bias=False)
+                          (conv): DilatedDenseNet(
+                            (pad): ConstantPad2d(padding=(1, 1, 1, 0), value=0.0)
+                            (pad1): ConstantPad2d(padding=(0, 0, 19, 19), value=0.0)
+                            (conv1): Conv2d(256, 256, kernel_size=(39, 1), stride=(1, 1), groups=256, bias=False)
+                            (norm1): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+                            (prelu1): PReLU(num_parameters=256)
+                            (pad2): ConstantPad2d(padding=(0, 0, 38, 38), value=0.0)
+                            (conv2): Conv2d(512, 256, kernel_size=(39, 1), stride=(1, 1), dilation=(2, 1), groups=256, bias=False)
+                            (norm2): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=False)
+                            (prelu2): PReLU(num_parameters=256)
+                          )
+                        )
+                      )
+                      (norm2): CLayerNorm((256,), eps=1e-05, elementwise_affine=True)
+                      (conv2): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
+                    )
+                  )
+                  (layers): ModuleList(
+                    (0-23): 24 x FLASH_ShareA_FFConvM(
+                      (rotary_pos_emb): RotaryEmbedding()
+                      (dropout): Dropout(p=0.1, inplace=False)
+                      (to_hidden): FFConvM(
+                        (mdl): Sequential(
+                          (0): ScaleNorm()
+                          (1): Linear(in_features=512, out_features=2048, bias=True)
+                          (2): SiLU()
+                          (3): ConvModule(
+                            (sequential): Sequential(
+                              (0): Transpose()
+                              (1): DepthwiseConv1d(
+                                (conv): Conv1d(2048, 2048, kernel_size=(17,), stride=(1,), padding=(8,), groups=2048, bias=False)
+                              )
+                            )
+                          )
+                          (4): Dropout(p=0.1, inplace=False)
+                        )
+                      )
+                      (to_qk): FFConvM(
+                        (mdl): Sequential(
+                          (0): ScaleNorm()
+                          (1): Linear(in_features=512, out_features=128, bias=True)
+                          (2): SiLU()
+                          (3): ConvModule(
+                            (sequential): Sequential(
+                              (0): Transpose()
+                              (1): DepthwiseConv1d(
+                                (conv): Conv1d(128, 128, kernel_size=(17,), stride=(1,), padding=(8,), groups=128, bias=False)
+                              )
+                            )
+                          )
+                          (4): Dropout(p=0.1, inplace=False)
+                        )
+                      )
+                      (qk_offset_scale): OffsetScale()
+                      (to_out): FFConvM(
+                        (mdl): Sequential(
+                          (0): ScaleNorm()
+                          (1): Linear(in_features=1024, out_features=512, bias=True)
+                          (2): SiLU()
+                          (3): ConvModule(
+                            (sequential): Sequential(
+                              (0): Transpose()
+                              (1): DepthwiseConv1d(
+                                (conv): Conv1d(512, 512, kernel_size=(17,), stride=(1,), padding=(8,), groups=512, bias=False)
+                              )
+                            )
+                          )
+                          (4): Dropout(p=0.1, inplace=False)
+                        )
+                      )
+                      (gateActivate): Sigmoid()
+                    )
+                  )
+                )
+                (norm): LayerNorm(
+                  (norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
+                )
+              )
+            )
+            (intra_norm): GroupNorm(1, 512, eps=1e-08, affine=True)
+          )
+        )
+        (conv1d_out): Conv1d(512, 512, kernel_size=(1,), stride=(1,))
+        (conv1_decoder): Conv1d(512, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (prelu): PReLU(num_parameters=1)
+        (activation): ReLU()
+        (output): Sequential(
+          (0): Conv1d(512, 512, kernel_size=(1,), stride=(1,))
+          (1): Tanh()
+        )
+        (output_gate): Sequential(
+          (0): Conv1d(512, 512, kernel_size=(1,), stride=(1,))
+          (1): Sigmoid()
+        )
+      )
+      (av_conv): Conv1d(768, 512, kernel_size=(1,), stride=(1,))
+    )
+    (decoder): Decoder(
+      (basis_signals): Linear(in_features=512, out_features=16, bias=False)
+    )
+  )
+  (ref_encoder): Visual_encoder(
+    (v_frontend): VisualFrontend(
+      (frontend3D): Sequential(
+        (0): Conv3d(1, 64, kernel_size=(5, 7, 7), stride=(1, 2, 2), padding=(2, 3, 3), bias=False)
+        (1): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        (2): ReLU()
+        (3): MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1), dilation=1, ceil_mode=False)
+      )
+      (resnet): ResNet(
+        (layer1): ResNetLayer(
+          (conv1a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
+          (outbna): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer2): ResNetLayer(
+          (conv1a): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer3): ResNetLayer(
+          (conv1a): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer4): ResNetLayer(
+          (conv1a): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (avgpool): AvgPool2d(kernel_size=(4, 4), stride=(1, 1), padding=0)
+      )
+    )
+    (v_ds): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+    (visual_conv): Sequential(
+      (0): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (1): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (2): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (3): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (4): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+    )
+  )
+)
+
+Total number of parameters: 68516407 
+
+
+Total number of trainable parameters: 57331303 
+
+dlcf4k2knsh01f6k-master-0:28:28 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlcf4k2knsh01f6k-master-0:28:28 [0] NCCL INFO Bootstrap : Using eth0:22.6.223.106<0>
+dlcf4k2knsh01f6k-master-0:28:28 [0] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlcf4k2knsh01f6k-master-0:28:28 [0] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
+dlcf4k2knsh01f6k-master-0:28:28 [0] NCCL INFO cudaDriverVersion 11040
+dlcf4k2knsh01f6k-master-0:29:29 [1] NCCL INFO cudaDriverVersion 11040
+NCCL version 2.20.5+cuda11.8
+dlcf4k2knsh01f6k-master-0:29:29 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlcf4k2knsh01f6k-master-0:29:29 [1] NCCL INFO Bootstrap : Using eth0:22.6.223.106<0>
+dlcf4k2knsh01f6k-master-0:29:29 [1] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlcf4k2knsh01f6k-master-0:29:29 [1] NCCL INFO NET/Plugin : dlerror=libnccl-net-none.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net-none.so), using internal implementation
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO NCCL_IB_HCA set to mlx5
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO NCCL_IB_HCA set to mlx5
+libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhfi1verbs-rdmav25.so': libhfi1verbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'librxe-rdmav25.so': librxe-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmthca-rdmav25.so': libmthca-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libvmw_pvrdma-rdmav25.so': libvmw_pvrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libhns-rdmav25.so': libhns-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libipathverbs-rdmav25.so': libipathverbs-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libsiw-rdmav25.so': libsiw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libbnxt_re-rdmav25.so': libbnxt_re-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libocrdma-rdmav25.so': libocrdma-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libmlx4-rdmav25.so': libmlx4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libqedr-rdmav25.so': libqedr-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libcxgb4-rdmav25.so': libcxgb4-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libi40iw-rdmav25.so': libi40iw-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
+libibverbs: Warning: couldn't load driver 'libefa-rdmav25.so': libefa-rdmav25.so: cannot open shared object file: No such file or directory
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth0:22.6.223.106<0>
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO Using non-device net plugin version 0
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO Using network IB
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth0:22.6.223.106<0>
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Using non-device net plugin version 0
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Using network IB
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO comm 0xf843110 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 10 commId 0x10c1230423267b73 - Init START
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO comm 0xdeaf580 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 20 commId 0x10c1230423267b73 - Init START
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO Setting affinity for GPU 1 to 0fff
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Setting affinity for GPU 0 to 0fff
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO comm 0xdeaf580 rank 1 nRanks 2 nNodes 1 localRanks 2 localRank 1 MNNVL 0
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO comm 0xf843110 rank 0 nRanks 2 nNodes 1 localRanks 2 localRank 0 MNNVL 0
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Channel 00/04 :    0   1
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO Trees [0] -1/-1/-1->1->0 [1] 0/-1/-1->1->-1 [2] -1/-1/-1->1->0 [3] 0/-1/-1->1->-1
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Channel 01/04 :    0   1
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO P2P Chunksize set to 524288
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Channel 02/04 :    0   1
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Channel 03/04 :    0   1
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] -1/-1/-1->0->1 [2] 1/-1/-1->0->-1 [3] -1/-1/-1->0->1
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO P2P Chunksize set to 524288
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO Channel 00/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO Channel 01/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO Channel 03/0 : 1[1] -> 0[0] via P2P/IPC/read
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/IPC/read
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO Connected all rings
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO Connected all trees
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Connected all rings
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO 4 coll channels, 0 collnet channels, 0 nvls channels, 4 p2p channels, 4 p2p channels per peer
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO Connected all trees
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 512 | 512
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO 4 coll channels, 0 collnet channels, 0 nvls channels, 4 p2p channels, 4 p2p channels per peer
+dlcf4k2knsh01f6k-master-0:28:47 [0] NCCL INFO comm 0xf843110 rank 0 nranks 2 cudaDev 0 nvmlDev 0 busId 10 commId 0x10c1230423267b73 - Init COMPLETE
+dlcf4k2knsh01f6k-master-0:29:48 [1] NCCL INFO comm 0xdeaf580 rank 1 nranks 2 cudaDev 1 nvmlDev 1 busId 20 commId 0x10c1230423267b73 - Init COMPLETE
+Start new training from scratch
+[rank0]:[W1112 09:29:26.544802480 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank1]:[W1112 09:29:26.544925757 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+Train Summary | End of Epoch 1 | Time 43669.98s | Train Loss 1.247
+Valid Summary | End of Epoch 1 | Time 901.08s | Valid Loss -1.256
+Test Summary | End of Epoch 1 | Time 541.46s | Test Loss -1.199
+Fund new best model, dict saved
+Train Summary | End of Epoch 2 | Time 43713.98s | Train Loss -2.610
+Valid Summary | End of Epoch 2 | Time 901.18s | Valid Loss -3.806
+Test Summary | End of Epoch 2 | Time 540.83s | Test Loss -3.796
+Fund new best model, dict saved
+Train Summary | End of Epoch 3 | Time 25073.10s | Train Loss -4.489
+Valid Summary | End of Epoch 3 | Time 450.91s | Valid Loss -5.026
+Test Summary | End of Epoch 3 | Time 271.13s | Test Loss -5.058
+Fund new best model, dict saved
+Train Summary | End of Epoch 4 | Time 25054.36s | Train Loss -5.677
+Valid Summary | End of Epoch 4 | Time 450.08s | Valid Loss -6.083
+Test Summary | End of Epoch 4 | Time 270.47s | Test Loss -6.067
+Fund new best model, dict saved
+Train Summary | End of Epoch 5 | Time 25076.36s | Train Loss -6.544
+Valid Summary | End of Epoch 5 | Time 450.20s | Valid Loss -6.711
+Test Summary | End of Epoch 5 | Time 270.51s | Test Loss -6.693
+Fund new best model, dict saved
+Train Summary | End of Epoch 6 | Time 25075.03s | Train Loss -7.214
+Valid Summary | End of Epoch 6 | Time 450.78s | Valid Loss -7.022
+Test Summary | End of Epoch 6 | Time 270.56s | Test Loss -7.022
+Fund new best model, dict saved
+Train Summary | End of Epoch 7 | Time 25072.53s | Train Loss -7.771
+Valid Summary | End of Epoch 7 | Time 450.92s | Valid Loss -7.599
+Test Summary | End of Epoch 7 | Time 270.66s | Test Loss -7.569
+Fund new best model, dict saved
+Train Summary | End of Epoch 8 | Time 25091.66s | Train Loss -8.231
+Valid Summary | End of Epoch 8 | Time 450.72s | Valid Loss -7.935
+Test Summary | End of Epoch 8 | Time 270.46s | Test Loss -7.773
+Fund new best model, dict saved
+Train Summary | End of Epoch 9 | Time 25093.91s | Train Loss -8.638
+Valid Summary | End of Epoch 9 | Time 450.64s | Valid Loss -8.063
+Test Summary | End of Epoch 9 | Time 270.51s | Test Loss -7.896
+Fund new best model, dict saved
+Train Summary | End of Epoch 10 | Time 25096.69s | Train Loss -8.994
+Valid Summary | End of Epoch 10 | Time 450.53s | Valid Loss -8.630
+Test Summary | End of Epoch 10 | Time 270.49s | Test Loss -8.485
+Fund new best model, dict saved
+Train Summary | End of Epoch 11 | Time 25093.05s | Train Loss -9.327
+Valid Summary | End of Epoch 11 | Time 450.49s | Valid Loss -8.803
+Test Summary | End of Epoch 11 | Time 270.54s | Test Loss -8.571
+Fund new best model, dict saved
+Train Summary | End of Epoch 12 | Time 25090.36s | Train Loss -9.571
+Valid Summary | End of Epoch 12 | Time 450.87s | Valid Loss -8.775
+Test Summary | End of Epoch 12 | Time 270.66s | Test Loss -8.658
+Train Summary | End of Epoch 13 | Time 25087.64s | Train Loss -9.798
+Valid Summary | End of Epoch 13 | Time 450.32s | Valid Loss -9.016
+Test Summary | End of Epoch 13 | Time 270.48s | Test Loss -8.783
+Fund new best model, dict saved
+Train Summary | End of Epoch 14 | Time 25089.46s | Train Loss -10.035
+Valid Summary | End of Epoch 14 | Time 451.11s | Valid Loss -9.213
+Test Summary | End of Epoch 14 | Time 271.00s | Test Loss -9.057
+Fund new best model, dict saved
+Train Summary | End of Epoch 15 | Time 25087.48s | Train Loss -10.248
+Valid Summary | End of Epoch 15 | Time 450.58s | Valid Loss -9.285
+Test Summary | End of Epoch 15 | Time 270.56s | Test Loss -9.162
+Fund new best model, dict saved
+Train Summary | End of Epoch 16 | Time 25086.70s | Train Loss -10.433
+Valid Summary | End of Epoch 16 | Time 449.93s | Valid Loss -9.573
+Test Summary | End of Epoch 16 | Time 270.22s | Test Loss -9.301
+Fund new best model, dict saved
+Train Summary | End of Epoch 17 | Time 25069.00s | Train Loss -10.590
+Valid Summary | End of Epoch 17 | Time 450.28s | Valid Loss -9.660
+Test Summary | End of Epoch 17 | Time 270.11s | Test Loss -9.497
+Fund new best model, dict saved
+Train Summary | End of Epoch 18 | Time 25070.59s | Train Loss -10.769
+Valid Summary | End of Epoch 18 | Time 449.77s | Valid Loss -9.758
+Test Summary | End of Epoch 18 | Time 270.33s | Test Loss -9.576
+Fund new best model, dict saved
+Train Summary | End of Epoch 19 | Time 25083.60s | Train Loss -10.908
+Valid Summary | End of Epoch 19 | Time 450.18s | Valid Loss -9.806
+Test Summary | End of Epoch 19 | Time 270.57s | Test Loss -9.576
+Fund new best model, dict saved
+Train Summary | End of Epoch 20 | Time 25077.18s | Train Loss -11.032
+Valid Summary | End of Epoch 20 | Time 450.19s | Valid Loss -9.904
+Test Summary | End of Epoch 20 | Time 270.30s | Test Loss -9.614
+Fund new best model, dict saved
+Train Summary | End of Epoch 21 | Time 25076.42s | Train Loss -11.159
+Valid Summary | End of Epoch 21 | Time 450.44s | Valid Loss -9.932
+Test Summary | End of Epoch 21 | Time 270.29s | Test Loss -9.735
+Fund new best model, dict saved
+Train Summary | End of Epoch 22 | Time 25071.05s | Train Loss -11.284
+Valid Summary | End of Epoch 22 | Time 449.99s | Valid Loss -10.161
+Test Summary | End of Epoch 22 | Time 270.03s | Test Loss -9.847
+Fund new best model, dict saved
+Train Summary | End of Epoch 23 | Time 25179.65s | Train Loss -11.413
+Valid Summary | End of Epoch 23 | Time 451.74s | Valid Loss -10.168
+Test Summary | End of Epoch 23 | Time 271.32s | Test Loss -10.012
+Fund new best model, dict saved
+Train Summary | End of Epoch 24 | Time 25184.85s | Train Loss -11.510
+Valid Summary | End of Epoch 24 | Time 450.85s | Valid Loss -10.270
+Test Summary | End of Epoch 24 | Time 270.98s | Test Loss -9.983
+Fund new best model, dict saved
+Train Summary | End of Epoch 25 | Time 25186.36s | Train Loss -11.605
+Valid Summary | End of Epoch 25 | Time 451.40s | Valid Loss -10.422
+Test Summary | End of Epoch 25 | Time 271.20s | Test Loss -10.065
+Fund new best model, dict saved
+Train Summary | End of Epoch 26 | Time 25201.47s | Train Loss -11.704
+Valid Summary | End of Epoch 26 | Time 453.12s | Valid Loss -10.403
+Test Summary | End of Epoch 26 | Time 272.61s | Test Loss -10.034
+Train Summary | End of Epoch 27 | Time 25220.07s | Train Loss -11.780
+Valid Summary | End of Epoch 27 | Time 451.85s | Valid Loss -10.491
+Test Summary | End of Epoch 27 | Time 271.49s | Test Loss -10.278
+Fund new best model, dict saved
+Train Summary | End of Epoch 28 | Time 25198.71s | Train Loss -11.887
+Valid Summary | End of Epoch 28 | Time 451.45s | Valid Loss -10.491
+Test Summary | End of Epoch 28 | Time 271.39s | Test Loss -10.255
+Fund new best model, dict saved
+Train Summary | End of Epoch 29 | Time 25196.59s | Train Loss -11.961
+Valid Summary | End of Epoch 29 | Time 452.05s | Valid Loss -10.628
+Test Summary | End of Epoch 29 | Time 271.64s | Test Loss -10.229
+Fund new best model, dict saved
+Train Summary | End of Epoch 30 | Time 25177.23s | Train Loss -12.047
+Valid Summary | End of Epoch 30 | Time 451.87s | Valid Loss -10.646
+Test Summary | End of Epoch 30 | Time 271.62s | Test Loss -10.514
+Fund new best model, dict saved
+Train Summary | End of Epoch 31 | Time 25189.61s | Train Loss -12.115
+Valid Summary | End of Epoch 31 | Time 451.54s | Valid Loss -10.731
+Test Summary | End of Epoch 31 | Time 271.47s | Test Loss -10.316
+Fund new best model, dict saved
+Train Summary | End of Epoch 32 | Time 25192.95s | Train Loss -12.189
+Valid Summary | End of Epoch 32 | Time 451.81s | Valid Loss -10.682
+Test Summary | End of Epoch 32 | Time 271.43s | Test Loss -10.461
+Train Summary | End of Epoch 33 | Time 25213.18s | Train Loss -12.261
+Valid Summary | End of Epoch 33 | Time 452.28s | Valid Loss -10.742
+Test Summary | End of Epoch 33 | Time 271.82s | Test Loss -10.376
+Fund new best model, dict saved
+Train Summary | End of Epoch 34 | Time 25239.52s | Train Loss -12.326
+Valid Summary | End of Epoch 34 | Time 452.05s | Valid Loss -10.735
+Test Summary | End of Epoch 34 | Time 271.85s | Test Loss -10.476
+Train Summary | End of Epoch 35 | Time 25201.77s | Train Loss -12.387
+Valid Summary | End of Epoch 35 | Time 452.37s | Valid Loss -10.790
+Test Summary | End of Epoch 35 | Time 271.61s | Test Loss -10.545
+Fund new best model, dict saved
+Train Summary | End of Epoch 36 | Time 25214.39s | Train Loss -12.448
+Valid Summary | End of Epoch 36 | Time 452.24s | Valid Loss -10.900
+Test Summary | End of Epoch 36 | Time 271.32s | Test Loss -10.561
+Fund new best model, dict saved
+Train Summary | End of Epoch 37 | Time 25189.86s | Train Loss -12.525
+Valid Summary | End of Epoch 37 | Time 451.71s | Valid Loss -10.881
+Test Summary | End of Epoch 37 | Time 271.45s | Test Loss -10.628
+Train Summary | End of Epoch 38 | Time 25183.78s | Train Loss -12.573
+Valid Summary | End of Epoch 38 | Time 451.63s | Valid Loss -10.884
+Test Summary | End of Epoch 38 | Time 271.73s | Test Loss -10.616
+Train Summary | End of Epoch 39 | Time 25016.31s | Train Loss -12.616
+Valid Summary | End of Epoch 39 | Time 450.59s | Valid Loss -10.945
+Test Summary | End of Epoch 39 | Time 270.98s | Test Loss -10.626
+Fund new best model, dict saved
+Train Summary | End of Epoch 40 | Time 25025.16s | Train Loss -12.672
+Valid Summary | End of Epoch 40 | Time 449.51s | Valid Loss -10.916
+Test Summary | End of Epoch 40 | Time 270.11s | Test Loss -10.699
+Train Summary | End of Epoch 41 | Time 24994.60s | Train Loss -12.720
+Valid Summary | End of Epoch 41 | Time 449.37s | Valid Loss -10.990
+Test Summary | End of Epoch 41 | Time 270.28s | Test Loss -10.662
+Fund new best model, dict saved
+Train Summary | End of Epoch 42 | Time 25011.89s | Train Loss -12.777
+Valid Summary | End of Epoch 42 | Time 449.97s | Valid Loss -10.992
+Test Summary | End of Epoch 42 | Time 270.13s | Test Loss -10.579
+Fund new best model, dict saved
+Train Summary | End of Epoch 43 | Time 25003.18s | Train Loss -12.816
+Valid Summary | End of Epoch 43 | Time 449.46s | Valid Loss -11.100
+Test Summary | End of Epoch 43 | Time 270.14s | Test Loss -10.774
+Fund new best model, dict saved
+Train Summary | End of Epoch 44 | Time 25008.71s | Train Loss -12.872
+Valid Summary | End of Epoch 44 | Time 449.52s | Valid Loss -11.058
+Test Summary | End of Epoch 44 | Time 270.12s | Test Loss -10.800
+Train Summary | End of Epoch 45 | Time 25001.41s | Train Loss -12.915
+Valid Summary | End of Epoch 45 | Time 449.84s | Valid Loss -11.000
+Test Summary | End of Epoch 45 | Time 270.30s | Test Loss -10.702
+Train Summary | End of Epoch 46 | Time 25044.45s | Train Loss -12.961
+Valid Summary | End of Epoch 46 | Time 449.92s | Valid Loss -11.090
+Test Summary | End of Epoch 46 | Time 270.63s | Test Loss -10.808
+Train Summary | End of Epoch 47 | Time 25096.63s | Train Loss -13.003
+Valid Summary | End of Epoch 47 | Time 450.40s | Valid Loss -11.108
+Test Summary | End of Epoch 47 | Time 270.46s | Test Loss -10.729
+Fund new best model, dict saved
+Train Summary | End of Epoch 48 | Time 25066.48s | Train Loss -13.052
+Valid Summary | End of Epoch 48 | Time 450.61s | Valid Loss -11.084
+Test Summary | End of Epoch 48 | Time 270.58s | Test Loss -10.807
+Train Summary | End of Epoch 49 | Time 25060.37s | Train Loss -13.081
+Valid Summary | End of Epoch 49 | Time 449.97s | Valid Loss -11.180
+Test Summary | End of Epoch 49 | Time 270.10s | Test Loss -10.787
+Fund new best model, dict saved
+Avg SISNR:i tensor([15.5134], device='cuda:0')
+Avg SNRi: 15.982982163172778
+Avg PESQi: 0.9377079456647237
+Avg STOIi: 0.3805843040861404

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/tensorboard/events.out.tfevents.1731374941.dlcf4k2knsh01f6k-master-0.28.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03b88a1a8717ab1bfb7de8b7da6d0c74f3bcd3cc66acb1b454503c393becb878
+size 384

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/tensorboard/events.out.tfevents.1731483665.dlcf4k2knsh01f6k-master-0.26.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcabdda15ce5d977fe7ee92a657542541518c7c02c24dcd6682a80cb12656321
+size 88

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/tensorboard/events.out.tfevents.1731487228.dlcf4k2knsh01f6k-master-0.26.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73a06be5c174f7ac57d165c3b995466916a10a0a6cc5a588996768b86e868388
+size 88

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/tensorboard/events.out.tfevents.1731492013.dlcf4k2knsh01f6k-master-0.26.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a87252d8a3672fe1cd328740d19bc87965d34221282468aad1f18527994f85c
+size 88

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/tensorboard/events.out.tfevents.1731546582.dlc199i687psn18d-master-0.26.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab592c940a0c4587bd3f6d3808883dcc4c0e948d4d46105c28a3a772cacbcbb9
+size 88

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/tensorboard/events.out.tfevents.1731548123.dlc9mw1l3osem0g9-master-0.1183013.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfbc0bc93f4e00ea6f68dc3af86e916bfe6795e3d10072e2cbcfb000224d8572
+size 88

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/tensorboard/events.out.tfevents.1731548338.dlc9mw1l3osem0g9-master-0.1187202.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:011a3bd08cf6325d461eef7d830aa7b07e77f77751efbbd4b00ce88dde2ca762
+size 88

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/tensorboard/events.out.tfevents.1731557306.dlc1yk2tc721dlue-master-0.26.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05641bb5d193dd14babc4450c0aac41eb299ab64ad1035f0621804a728895b33
+size 3048

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/tensorboard/events.out.tfevents.1732075802.dlc1g7yr5z0x4h2g-master-0.26.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9c8d8e2ff4cceb649d8a7ac6fe0ba2c19b8ba81ea36d3bb14aa9fd0a00588a2
+size 2456

checkpoints/log_VoxCeleb2_lip_mossformer2_3spk/tensorboard/events.out.tfevents.1732583305.dlc1gzcv7row61qr-master-0.26.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dbf2602a6c0dc68e378107dd8a606e67da383caff0a65179e37243954bff650
+size 1716