train.12.log

# Running on cnode7-008
# Started at Thu Nov 28 15:14:44 CST 2024
# SLURMD_NODENAME=cnode7-008
# SLURM_CLUSTER_NAME=slurm
# SLURM_CONF=/cm/shared/apps/slurm/var/etc/slurm/slurm.conf
# SLURM_CPUS_ON_NODE=224
# SLURM_CPUS_PER_TASK=128
# SLURM_EXPORT_ENV=PATH
# SLURM_GET_USER_ENV=1
# SLURM_GPUS_ON_NODE=8
# SLURM_GPUS_PER_NODE=8
# SLURM_GTIDS=0
# SLURM_JOBID=4946
# SLURM_JOB_CPUS_PER_NODE='224(x2)'
# SLURM_JOB_END_TIME=1764314080
# SLURM_JOB_GID=1026
# SLURM_JOB_GPUS=0,1,2,3,4,5,6,7
# SLURM_JOB_ID=4946
# SLURM_JOB_NAME=exp_owsm/s2t_train_05b_ds_raw_bpe50000/train.log
# SLURM_JOB_NODELIST='cnode7-[008-009]'
# SLURM_JOB_NUM_NODES=2
# SLURM_JOB_PARTITION=p2
# SLURM_JOB_QOS=normal
# SLURM_JOB_START_TIME=1732778080
# SLURM_JOB_UID=1026
# SLURM_JOB_USER=williamchen
# SLURM_LOCALID=0
# SLURM_MEM_PER_NODE=2048000
# SLURM_NNODES=2
# SLURM_NODEID=0
# SLURM_NODELIST='cnode7-[008-009]'
# SLURM_NODE_ALIASES='(null)'
# SLURM_OPEN_MODE=a
# SLURM_PRIO_PROCESS=0
# SLURM_PROCID=0
# SLURM_SUBMIT_DIR=/mnt/home/williamchen/espnet/egs2/owsm_v3.1/s2t1
# SLURM_SUBMIT_HOST=154-T2-P1-NVR
# SLURM_TASKS_PER_NODE='1(x2)'
# SLURM_TASK_PID=1503870
# SLURM_TOPOLOGY_ADDR=cnode7-008
# SLURM_TOPOLOGY_ADDR_PATTERN=node
# SLURM_WORKING_CLUSTER=slurm:154-T2-P1-NVR:6817:9984:109
# srun --export=ALL python3 -m espnet2.bin.s2t_train --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp_owsm/s2t_train_05b_ds_raw_bpe50000 --config conf/train_05b_ds.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp_owsm/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/wav.scp,speech,kaldi_ark --train_shape_file exp_owsm/s2t_stats_raw_bpe50000/splits8/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text.prev,text_prev,text --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text.ctc,text_ctc,text --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text,text,text --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/text_prev_shape --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/text_ctc_shape --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/text_shape --ngpu 8 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///mnt/home/williamchen/espnet/egs2/owsm_v3.1/s2t1/exp_owsm/s2t_train_05b_ds_raw_bpe50000/.dist_init_4c8213f8-bbff-47c8-873a-9e190d4afe53 
[2024-11-28 15:14:54,974] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:14:57,866] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
/mnt/home/williamchen/espnet/tools/miniconda/envs/espnet/bin/python3 /mnt/home/williamchen/espnet/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp_owsm/s2t_train_05b_ds_raw_bpe50000 --config conf/train_05b_ds.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp_owsm/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/wav.scp,speech,kaldi_ark --train_shape_file exp_owsm/s2t_stats_raw_bpe50000/splits8/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text.prev,text_prev,text --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text.ctc,text_ctc,text --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text,text,text --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/text_prev_shape --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/text_ctc_shape --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/text_shape --ngpu 8 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///mnt/home/williamchen/espnet/egs2/owsm_v3.1/s2t1/exp_owsm/s2t_train_05b_ds_raw_bpe50000/.dist_init_4c8213f8-bbff-47c8-873a-9e190d4afe53
/mnt/home/williamchen/espnet/tools/miniconda/envs/espnet/bin/python3 /mnt/home/williamchen/espnet/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp_owsm/s2t_train_05b_ds_raw_bpe50000 --config conf/train_05b_ds.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp_owsm/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/wav.scp,speech,kaldi_ark --train_shape_file exp_owsm/s2t_stats_raw_bpe50000/splits8/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text.prev,text_prev,text --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text.ctc,text_ctc,text --fold_length 150 --train_data_path_and_name_and_type exp_owsm/s2t_stats_raw_bpe50000/splits8/text,text,text --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/text_prev_shape --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/text_ctc_shape --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp_owsm/s2t_stats_raw_bpe50000/valid/text_shape --ngpu 8 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///mnt/home/williamchen/espnet/egs2/owsm_v3.1/s2t1/exp_owsm/s2t_train_05b_ds_raw_bpe50000/.dist_init_4c8213f8-bbff-47c8-873a-9e190d4afe53
[2024-11-28 15:15:17,972] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:15,301] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:15,700] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:18,441] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:15,749] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:18,462] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:18,545] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:18,637] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:15,982] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:18,765] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:18,796] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:18,850] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:16,183] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:16,210] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:16,292] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-11-28 15:15:16,330] [INFO] [real_accelerator.py:219:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[W1128 15:15:22.590933009 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:25.162658043 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:26.368137303 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:23.025406997 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:23.042933992 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:23.052782040 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:26.666921758 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:26.667129398 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:26.684706832 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:23.474303364 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:26.885105709 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:26.887277046 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:23.504727705 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:23.504793967 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[W1128 15:15:26.907160538 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[cnode7-008:0/16] 2024-11-28 15:15:26,589 (s2t:462) INFO: Vocabulary size: 50002
[W1128 15:15:23.523326241 Utils.hpp:164] Warning: Environment variable NCCL_BLOCKING_WAIT is deprecated; use TORCH_NCCL_BLOCKING_WAIT instead (function operator())
[cnode7-008:0/16] 2024-11-28 15:15:33,575 (abs_task:1383) INFO: pytorch.version=2.4.0+cu121, cuda.available=True, cudnn.version=90100, cudnn.benchmark=False, cudnn.deterministic=True
[cnode7-008:0/16] 2024-11-28 15:15:33,578 (abs_task:1384) INFO: Model structure:
ESPnetS2TModel(
  (frontend): DefaultFrontend(
    (stft): Stft(n_fft=512, win_length=400, hop_length=160, center=True, normalized=False, onesided=True)
    (frontend): Frontend()
    (logmel): LogMel(sr=16000, n_fft=512, n_mels=80, fmin=0, fmax=8000.0, htk=False)
  )
  (specaug): SpecAug(
    (freq_mask): MaskAlongAxis(mask_width_range=[0, 27], num_mask=2, axis=freq)
    (time_mask): MaskAlongAxisVariableMaxWidth(mask_width_ratio_range=[0.0, 0.05], num_mask=10, axis=time)
  )
  (normalize): GlobalMVN(stats_file=exp_owsm/s2t_stats_raw_bpe50000/train/feats_stats.npz, norm_means=True, norm_vars=True)
  (encoder): TransformerEncoder(
    (embed): Conv2dSubsampling(
      (conv): Sequential(
        (0): Conv2d(1, 2048, kernel_size=(3, 3), stride=(2, 2))
        (1): ReLU()
        (2): Conv2d(2048, 2048, kernel_size=(3, 3), stride=(2, 2))
        (3): ReLU()
      )
      (out): Sequential(
        (0): Linear(in_features=38912, out_features=2048, bias=True)
        (1): PositionalEncoding(
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
    )
    (encoders): MultiSequential(
      (0): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (1): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (2): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (3): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (4): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (5): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (6): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (7): EncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
    )
    (after_norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
  )
  (decoder): TransformerDecoder(
    (embed): Sequential(
      (0): Embedding(50002, 2048)
      (1): PositionalEncoding(
        (dropout): Dropout(p=0.1, inplace=False)
      )
    )
    (after_norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
    (output_layer): Linear(in_features=2048, out_features=50002, bias=True)
    (decoders): MultiSequential(
      (0): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (1): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (2): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (3): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (4): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (5): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (6): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (7): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_k): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_v): Linear(in_features=2048, out_features=2048, bias=True)
          (linear_out): Linear(in_features=2048, out_features=2048, bias=True)
          (dropout): Identity()
          (q_norm): Identity()
          (k_norm): Identity()
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=2048, out_features=8192, bias=True)
          (w_2): Linear(in_features=8192, out_features=2048, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
    )
  )
  (criterion_att): LabelSmoothingLoss(
    (criterion): KLDivLoss()
  )
  (ctc): CTC(
    (ctc_lo): Linear(in_features=2048, out_features=50002, bias=True)
    (ctc_loss): CTCLoss()
  )
)

Model summary:
    Class Name: ESPnetS2TModel
    Total Number of model parameters: 1.36 B
    Number of trainable parameters: 1.36 B (100.0%)
    Size: 5.46 GB
    Type: torch.float32
[cnode7-008:0/16] 2024-11-28 15:15:33,578 (abs_task:1387) INFO: Optimizer:
Adadelta (
Parameter Group 0
    capturable: False
    differentiable: False
    eps: 1e-06
    foreach: None
    lr: 1.0
    maximize: False
    rho: 0.9
    weight_decay: 0
)
[cnode7-008:0/16] 2024-11-28 15:15:33,578 (abs_task:1388) INFO: Scheduler: None
[cnode7-008:0/16] 2024-11-28 15:15:33,580 (abs_task:1397) INFO: Saving the configuration in exp_owsm/s2t_train_05b_ds_raw_bpe50000/config.yaml
[cnode7-008:0/16] 2024-11-28 15:15:38,133 (s2t:444) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:38,949] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:38,952] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:36,294] [INFO] [comm.py:652:init_distributed] cdb=None
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:36,297] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
[2024-11-28 15:15:36,299] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:36,300] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:39,051] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:39,052] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
[cnode7-008:0/16] 2024-11-28 15:15:39,071 (abs_task:1807) INFO: [valid] dataset:
ESPnetDataset(
  speech: {"path": "dump/raw/dev_v3/wav.scp", "type": "kaldi_ark"}
  text_prev: {"path": "dump/raw/dev_v3/text.prev", "type": "text"}
  text_ctc: {"path": "dump/raw/dev_v3/text.ctc", "type": "text"}
  text: {"path": "dump/raw/dev_v3/text", "type": "text"}
  preprocess: <espnet2.train.preprocessor.S2TPreprocessor object at 0x15523e0a9870>)
[cnode7-008:0/16] 2024-11-28 15:15:39,071 (abs_task:1808) INFO: [valid] Batch sampler: UnsortedBatchSampler(N-batch=74743, batch_size=16, key_file=exp_owsm/s2t_stats_raw_bpe50000/valid/speech_shape, 
[cnode7-008:0/16] 2024-11-28 15:15:39,077 (abs_task:1809) INFO: [valid] mini-batch sizes summary: N-batch=74743, mean=16.0, min=16, max=17
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:39,089] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:39,090] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:39,113] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:39,114] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:39,179] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:39,180] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:36,473] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:36,473] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:36,522] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:36,523] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[cnode7-008:0/16] 2024-11-28 15:15:39,327 (distributed_utils:129) WARNING: 
=================================================================
Found OMP_NUM_THREADS=1 in environment variables. With some advanced features, DeepSpeed may have heavy cpu workload so that OMP_NUM_THREADS=1 is not sufficient. Try to increase it in your path.sh 
=================================================================
[2024-11-28 15:15:39,327] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:39,328] [INFO] [logging.py:129:log_dist] [Rank 0] DeepSpeed info: version=0.15.3, git-hash=unknown, git-branch=unknown
[2024-11-28 15:15:39,328] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
cnode7-008:1504977:1504977 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504977:1504977 [0] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.170<0>
cnode7-008:1504977:1504977 [0] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-008:1504977:1504977 [0] NCCL INFO cudaDriverVersion 12020
NCCL version 2.20.5+cuda12.4
cnode7-008:1504978:1504978 [1] NCCL INFO cudaDriverVersion 12020
cnode7-008:1504978:1504978 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504978:1504978 [1] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.170<0>
cnode7-008:1504978:1504978 [1] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-009:635262:635262 [5] NCCL INFO cudaDriverVersion 12020
cnode7-009:635262:635262 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635262:635262 [5] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.171<0>
cnode7-009:635262:635262 [5] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-008:1504982:1504982 [5] NCCL INFO cudaDriverVersion 12020
cnode7-008:1504982:1504982 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504982:1504982 [5] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.170<0>
cnode7-008:1504982:1504982 [5] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-008:1504984:1504984 [7] NCCL INFO cudaDriverVersion 12020
cnode7-008:1504984:1504984 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504984:1504984 [7] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.170<0>
cnode7-008:1504984:1504984 [7] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-009:635257:635257 [0] NCCL INFO cudaDriverVersion 12020
cnode7-009:635257:635257 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635257:635257 [0] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.171<0>
cnode7-009:635257:635257 [0] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-009:635264:635264 [7] NCCL INFO cudaDriverVersion 12020
cnode7-009:635264:635264 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635264:635264 [7] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.171<0>
cnode7-009:635264:635264 [7] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-008:1504980:1504980 [3] NCCL INFO cudaDriverVersion 12020
cnode7-008:1504980:1504980 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504980:1504980 [3] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.170<0>
cnode7-008:1504980:1504980 [3] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-008:1504983:1504983 [6] NCCL INFO cudaDriverVersion 12020
cnode7-008:1504983:1504983 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504983:1504983 [6] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.170<0>
cnode7-008:1504983:1504983 [6] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-009:635259:635259 [2] NCCL INFO cudaDriverVersion 12020
cnode7-009:635259:635259 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635259:635259 [2] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.171<0>
cnode7-009:635259:635259 [2] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:36,918] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:36,919] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
cnode7-008:1504978:1505583 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504983:1505585 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504982:1505587 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504984:1505584 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504977:1505582 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504980:1505586 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635260:635260 [3] NCCL INFO cudaDriverVersion 12020
cnode7-009:635260:635260 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635260:635260 [3] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.171<0>
cnode7-009:635260:635260 [3] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-008:1504978:1505583 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.170<0>
cnode7-008:1504978:1505583 [1] NCCL INFO Using non-device net plugin version 0
cnode7-008:1504978:1505583 [1] NCCL INFO Using network IB
cnode7-008:1504983:1505585 [6] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.170<0>
cnode7-008:1504983:1505585 [6] NCCL INFO Using non-device net plugin version 0
cnode7-008:1504983:1505585 [6] NCCL INFO Using network IB
cnode7-009:635259:636004 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635262:636000 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504982:1505587 [5] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.170<0>
cnode7-008:1504982:1505587 [5] NCCL INFO Using non-device net plugin version 0
cnode7-008:1504982:1505587 [5] NCCL INFO Using network IB
cnode7-008:1504984:1505584 [7] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.170<0>
cnode7-008:1504984:1505584 [7] NCCL INFO Using non-device net plugin version 0
cnode7-008:1504984:1505584 [7] NCCL INFO Using network IB
cnode7-009:635257:636003 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635264:636002 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504977:1505582 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.170<0>
cnode7-008:1504977:1505582 [0] NCCL INFO Using non-device net plugin version 0
cnode7-008:1504977:1505582 [0] NCCL INFO Using network IB
cnode7-008:1504980:1505586 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.170<0>
cnode7-008:1504980:1505586 [3] NCCL INFO Using non-device net plugin version 0
cnode7-008:1504980:1505586 [3] NCCL INFO Using network IB
cnode7-009:635259:636004 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.171<0>
cnode7-009:635259:636004 [2] NCCL INFO Using non-device net plugin version 0
cnode7-009:635259:636004 [2] NCCL INFO Using network IB
cnode7-009:635262:636000 [5] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.171<0>
cnode7-009:635262:636000 [5] NCCL INFO Using non-device net plugin version 0
cnode7-009:635262:636000 [5] NCCL INFO Using network IB
cnode7-009:635257:636003 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.171<0>
cnode7-009:635257:636003 [0] NCCL INFO Using non-device net plugin version 0
cnode7-009:635257:636003 [0] NCCL INFO Using network IB
cnode7-009:635264:636002 [7] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.171<0>
cnode7-009:635264:636002 [7] NCCL INFO Using non-device net plugin version 0
cnode7-009:635264:636002 [7] NCCL INFO Using network IB
cnode7-009:635260:636022 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635260:636022 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.171<0>
cnode7-009:635260:636022 [3] NCCL INFO Using non-device net plugin version 0
cnode7-009:635260:636022 [3] NCCL INFO Using network IB
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:39,946] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:39,948] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:37,258] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:37,260] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
cnode7-008:1504981:1504981 [4] NCCL INFO cudaDriverVersion 12020
cnode7-008:1504981:1504981 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504981:1504981 [4] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.170<0>
cnode7-008:1504981:1504981 [4] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-009:635261:635261 [4] NCCL INFO cudaDriverVersion 12020
cnode7-009:635261:635261 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635261:635261 [4] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.171<0>
cnode7-009:635261:635261 [4] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:38,095] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:38,097] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
cnode7-008:1504981:1505653 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635263:635263 [6] NCCL INFO cudaDriverVersion 12020
cnode7-009:635263:635263 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635263:635263 [6] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.171<0>
cnode7-009:635263:635263 [6] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
[2024-11-28 15:15:40,908] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:40,910] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
cnode7-009:635261:636079 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504979:1504979 [2] NCCL INFO cudaDriverVersion 12020
cnode7-008:1504979:1504979 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504979:1504979 [2] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.170<0>
cnode7-008:1504979:1504979 [2] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-009:635261:636079 [4] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.171<0>
cnode7-009:635261:636079 [4] NCCL INFO Using non-device net plugin version 0
cnode7-009:635261:636079 [4] NCCL INFO Using network IB
cnode7-008:1504981:1505653 [4] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.170<0>
cnode7-008:1504981:1505653 [4] NCCL INFO Using non-device net plugin version 0
cnode7-008:1504981:1505653 [4] NCCL INFO Using network IB
cnode7-009:635263:636084 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
/mnt/home/williamchen/espnet/espnet2/train/deepspeed_trainer.py:238: InstrumentationWarning: @typechecked only supports instrumenting functions wrapped with @classmethod, @staticmethod or @property -- not typechecking espnet2.train.deepspeed_trainer.DeepSpeedTrainer.valid_one_epoch
  def valid_one_epoch(
cnode7-009:635263:636084 [6] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.171<0>
cnode7-009:635263:636084 [6] NCCL INFO Using non-device net plugin version 0
cnode7-009:635263:636084 [6] NCCL INFO Using network IB
[2024-11-28 15:15:38,687] [INFO] [comm.py:652:init_distributed] cdb=None
[2024-11-28 15:15:38,688] [INFO] [config.py:733:__init__] Config mesh_device None world_size = 16
cnode7-008:1504979:1505667 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-008:1504979:1505667 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.170<0>
cnode7-008:1504979:1505667 [2] NCCL INFO Using non-device net plugin version 0
cnode7-008:1504979:1505667 [2] NCCL INFO Using network IB
cnode7-009:635258:635258 [1] NCCL INFO cudaDriverVersion 12020
cnode7-009:635258:635258 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635258:635258 [1] NCCL INFO Bootstrap : Using enp170s0np0:10.225.1.171<0>
cnode7-009:635258:635258 [1] NCCL INFO NET/Plugin : dlerror=libnccl-net.so: cannot open shared object file: No such file or directory No plugin found (libnccl-net.so), using internal implementation
cnode7-009:635258:636105 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to ^lo,docker,virbr,vmnet,vboxnet
cnode7-009:635258:636105 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/IB [1]mlx5_3:1/IB [2]mlx5_4:1/IB [3]mlx5_5:1/IB [4]mlx5_6:1/IB [5]mlx5_7:1/RoCE [6]mlx5_8:1/IB [7]mlx5_9:1/IB [8]mlx5_10:1/IB [RO]; OOB enp170s0np0:10.225.1.171<0>
cnode7-009:635258:636105 [1] NCCL INFO Using non-device net plugin version 0
cnode7-009:635258:636105 [1] NCCL INFO Using network IB
cnode7-008:1504979:1505667 [2] NCCL INFO comm 0x555563055650 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 52000 commId 0x90771211591af19e - Init START
cnode7-008:1504978:1505583 [1] NCCL INFO comm 0x555562abe5e0 rank 1 nranks 16 cudaDev 1 nvmlDev 1 busId 43000 commId 0x90771211591af19e - Init START
cnode7-008:1504983:1505585 [6] NCCL INFO comm 0x555562abd3b0 rank 6 nranks 16 cudaDev 6 nvmlDev 6 busId d1000 commId 0x90771211591af19e - Init START
cnode7-009:635259:636004 [2] NCCL INFO comm 0x5555694db980 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 52000 commId 0x90771211591af19e - Init START
cnode7-009:635262:636000 [5] NCCL INFO comm 0x55556255d900 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId c3000 commId 0x90771211591af19e - Init START
cnode7-008:1504981:1505653 [4] NCCL INFO comm 0x5555694d8a50 rank 4 nranks 16 cudaDev 4 nvmlDev 4 busId 9d000 commId 0x90771211591af19e - Init START
cnode7-008:1504982:1505587 [5] NCCL INFO comm 0x555564b79df0 rank 5 nranks 16 cudaDev 5 nvmlDev 5 busId c3000 commId 0x90771211591af19e - Init START
cnode7-008:1504984:1505584 [7] NCCL INFO comm 0x555572152360 rank 7 nranks 16 cudaDev 7 nvmlDev 7 busId df000 commId 0x90771211591af19e - Init START
cnode7-009:635263:636084 [6] NCCL INFO comm 0x555564b7b720 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId d1000 commId 0x90771211591af19e - Init START
cnode7-009:635257:636003 [0] NCCL INFO comm 0x5555bdc83010 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x90771211591af19e - Init START
cnode7-009:635264:636002 [7] NCCL INFO comm 0x5555638e85b0 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId df000 commId 0x90771211591af19e - Init START
cnode7-009:635261:636079 [4] NCCL INFO comm 0x5555694dc520 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId 9d000 commId 0x90771211591af19e - Init START
cnode7-008:1504977:1505582 [0] NCCL INFO comm 0x5555bdcea200 rank 0 nranks 16 cudaDev 0 nvmlDev 0 busId 1b000 commId 0x90771211591af19e - Init START
cnode7-008:1504980:1505586 [3] NCCL INFO comm 0x5555638e7630 rank 3 nranks 16 cudaDev 3 nvmlDev 3 busId 61000 commId 0x90771211591af19e - Init START
cnode7-009:635260:636022 [3] NCCL INFO comm 0x555564b7d7e0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 61000 commId 0x90771211591af19e - Init START
cnode7-009:635258:636105 [1] NCCL INFO comm 0x5555638e63a0 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 43000 commId 0x90771211591af19e - Init START
cnode7-008:1504978:1505583 [1] NCCL INFO Setting affinity for GPU 1 to ff,ffffffff,ffff0000,00000000,00ffffff,ffffffff
cnode7-008:1504978:1505583 [1] NCCL INFO NVLS multicast support is available on dev 1
cnode7-008:1504977:1505582 [0] NCCL INFO Setting affinity for GPU 0 to ff,ffffffff,ffff0000,00000000,00ffffff,ffffffff
cnode7-008:1504977:1505582 [0] NCCL INFO NVLS multicast support is available on dev 0
cnode7-008:1504980:1505586 [3] NCCL INFO Setting affinity for GPU 3 to ff,ffffffff,ffff0000,00000000,00ffffff,ffffffff
cnode7-008:1504980:1505586 [3] NCCL INFO NVLS multicast support is available on dev 3
cnode7-008:1504983:1505585 [6] NCCL INFO Setting affinity for GPU 6 to ffffffff,ffffff00,00000000,0000ffff,ffffffff,ff000000,00000000
cnode7-008:1504983:1505585 [6] NCCL INFO NVLS multicast support is available on dev 6
cnode7-008:1504984:1505584 [7] NCCL INFO Setting affinity for GPU 7 to ffffffff,ffffff00,00000000,0000ffff,ffffffff,ff000000,00000000
cnode7-008:1504984:1505584 [7] NCCL INFO NVLS multicast support is available on dev 7
cnode7-008:1504979:1505667 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffffffff,ffff0000,00000000,00ffffff,ffffffff
cnode7-008:1504979:1505667 [2] NCCL INFO NVLS multicast support is available on dev 2
cnode7-008:1504981:1505653 [4] NCCL INFO Setting affinity for GPU 4 to ffffffff,ffffff00,00000000,0000ffff,ffffffff,ff000000,00000000
cnode7-008:1504981:1505653 [4] NCCL INFO NVLS multicast support is available on dev 4
cnode7-008:1504982:1505587 [5] NCCL INFO Setting affinity for GPU 5 to ffffffff,ffffff00,00000000,0000ffff,ffffffff,ff000000,00000000
cnode7-008:1504982:1505587 [5] NCCL INFO NVLS multicast support is available on dev 5
cnode7-009:635262:636000 [5] NCCL INFO Setting affinity for GPU 5 to ffffffff,ffffff00,00000000,0000ffff,ffffffff,ff000000,00000000
cnode7-009:635262:636000 [5] NCCL INFO NVLS multicast support is available on dev 5
cnode7-009:635264:636002 [7] NCCL INFO Setting affinity for GPU 7 to ffffffff,ffffff00,00000000,0000ffff,ffffffff,ff000000,00000000
cnode7-009:635264:636002 [7] NCCL INFO NVLS multicast support is available on dev 7
cnode7-009:635259:636004 [2] NCCL INFO Setting affinity for GPU 2 to ff,ffffffff,ffff0000,00000000,00ffffff,ffffffff
cnode7-009:635259:636004 [2] NCCL INFO NVLS multicast support is available on dev 2
cnode7-009:635260:636022 [3] NCCL INFO Setting affinity for GPU 3 to ff,ffffffff,ffff0000,00000000,00ffffff,ffffffff
cnode7-009:635260:636022 [3] NCCL INFO NVLS multicast support is available on dev 3
cnode7-009:635261:636079 [4] NCCL INFO Setting affinity for GPU 4 to ffffffff,ffffff00,00000000,0000ffff,ffffffff,ff000000,00000000
cnode7-009:635261:636079 [4] NCCL INFO NVLS multicast support is available on dev 4
cnode7-009:635258:636105 [1] NCCL INFO Setting affinity for GPU 1 to ff,ffffffff,ffff0000,00000000,00ffffff,ffffffff
cnode7-009:635258:636105 [1] NCCL INFO NVLS multicast support is available on dev 1
cnode7-008:1504980:1505586 [3] NCCL INFO comm 0x5555638e7630 rank 3 nRanks 16 nNodes 2 localRanks 8 localRank 3 MNNVL 0
cnode7-008:1504980:1505586 [3] NCCL INFO NVLS Head  0:  0  8
cnode7-008:1504980:1505586 [3] NCCL INFO NVLS Head  1:  1  9
cnode7-008:1504980:1505586 [3] NCCL INFO NVLS Head  2:  2 10
cnode7-008:1504980:1505586 [3] NCCL INFO NVLS Head  3:  3 11
cnode7-008:1504980:1505586 [3] NCCL INFO NVLS Head  4:  4 12
cnode7-008:1504980:1505586 [3] NCCL INFO NVLS Head  5:  5 13
cnode7-008:1504980:1505586 [3] NCCL INFO NVLS Head  6:  6 14
cnode7-008:1504980:1505586 [3] NCCL INFO NVLS Head  7:  7 15
cnode7-008:1504980:1505586 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/11/-1->3->-1 [4] -1/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->11 [12] -1/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2
cnode7-008:1504980:1505586 [3] NCCL INFO P2P Chunksize set to 131072
cnode7-008:1504977:1505582 [0] NCCL INFO comm 0x5555bdcea200 rank 0 nRanks 16 nNodes 2 localRanks 8 localRank 0 MNNVL 0
cnode7-008:1504977:1505582 [0] NCCL INFO NVLS Head  0:  0  8
cnode7-008:1504977:1505582 [0] NCCL INFO NVLS Head  1:  1  9
cnode7-008:1504977:1505582 [0] NCCL INFO NVLS Head  2:  2 10
cnode7-008:1504977:1505582 [0] NCCL INFO NVLS Head  3:  3 11
cnode7-008:1504977:1505582 [0] NCCL INFO NVLS Head  4:  4 12
cnode7-008:1504977:1505582 [0] NCCL INFO NVLS Head  5:  5 13
cnode7-008:1504977:1505582 [0] NCCL INFO NVLS Head  6:  6 14
cnode7-008:1504977:1505582 [0] NCCL INFO NVLS Head  7:  7 15
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 00/16 :    0   7   6   5   4   3   2   1   9  10  11  12  13  14  15   8
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 01/16 :    0   8  15  14  13  12  11  10   9   1   2   3   4   5   6   7
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 02/16 :    0   7   6   5   4   3  11  12  13  14  15   8   9  10   2   1
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 03/16 :    0   1   2  10   9   8  15  14  13  12  11   3   4   5   6   7
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 04/16 :    0   7   6   5  13  14  15   8   9  10  11  12   4   3   2   1
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 05/16 :    0   1   2   3   4  12  11  10   9   8  15  14  13   5   6   7
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 06/16 :    0   7  15   8   9  10  11  12  13  14   6   5   4   3   2   1
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 07/16 :    0   1   2   3   4   5   6  14  13  12  11  10   9   8  15   7
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 08/16 :    0   7   6   5   4   3   2   1   9  10  11  12  13  14  15   8
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 09/16 :    0   8  15  14  13  12  11  10   9   1   2   3   4   5   6   7
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 10/16 :    0   7   6   5   4   3  11  12  13  14  15   8   9  10   2   1
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 11/16 :    0   1   2  10   9   8  15  14  13  12  11   3   4   5   6   7
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 12/16 :    0   7   6   5  13  14  15   8   9  10  11  12   4   3   2   1
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 13/16 :    0   1   2   3   4  12  11  10   9   8  15  14  13   5   6   7
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 14/16 :    0   7  15   8   9  10  11  12  13  14   6   5   4   3   2   1
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 15/16 :    0   1   2   3   4   5   6  14  13  12  11  10   9   8  15   7
cnode7-008:1504977:1505582 [0] NCCL INFO Trees [0] 1/8/-1->0->-1 [1] -1/-1/-1->0->7 [2] 1/-1/-1->0->7 [3] 1/-1/-1->0->7 [4] 1/-1/-1->0->7 [5] 1/-1/-1->0->7 [6] 1/-1/-1->0->7 [7] 1/-1/-1->0->7 [8] 1/-1/-1->0->8 [9] -1/-1/-1->0->7 [10] 1/-1/-1->0->7 [11] 1/-1/-1->0->7 [12] 1/-1/-1->0->7 [13] 1/-1/-1->0->7 [14] 1/-1/-1->0->7 [15] 1/-1/-1->0->7
cnode7-008:1504977:1505582 [0] NCCL INFO P2P Chunksize set to 131072
cnode7-009:635262:636000 [5] NCCL INFO comm 0x55556255d900 rank 13 nRanks 16 nNodes 2 localRanks 8 localRank 5 MNNVL 0
cnode7-009:635262:636000 [5] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->12 [2] 14/-1/-1->13->12 [3] 14/-1/-1->13->12 [4] 14/-1/-1->13->12 [5] 14/-1/-1->13->5 [6] -1/-1/-1->13->12 [7] 14/-1/-1->13->12 [8] 14/-1/-1->13->12 [9] 14/-1/-1->13->12 [10] 14/-1/-1->13->12 [11] 14/-1/-1->13->12 [12] 14/-1/-1->13->12 [13] 14/5/-1->13->-1 [14] -1/-1/-1->13->12 [15] 14/-1/-1->13->12
cnode7-009:635262:636000 [5] NCCL INFO P2P Chunksize set to 131072
cnode7-009:635264:636002 [7] NCCL INFO comm 0x5555638e85b0 rank 15 nRanks 16 nNodes 2 localRanks 8 localRank 7 MNNVL 0
cnode7-009:635264:636002 [7] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 8/-1/-1->15->14 [2] 8/-1/-1->15->14 [3] 8/-1/-1->15->14 [4] 8/-1/-1->15->14 [5] 8/-1/-1->15->14 [6] 8/-1/-1->15->14 [7] 8/-1/-1->15->7 [8] -1/-1/-1->15->14 [9] 8/-1/-1->15->14 [10] 8/-1/-1->15->14 [11] 8/-1/-1->15->14 [12] 8/-1/-1->15->14 [13] 8/-1/-1->15->14 [14] 8/-1/-1->15->14 [15] 8/7/-1->15->-1
cnode7-009:635264:636002 [7] NCCL INFO P2P Chunksize set to 131072
cnode7-009:635263:636084 [6] NCCL INFO Setting affinity for GPU 6 to ffffffff,ffffff00,00000000,0000ffff,ffffffff,ff000000,00000000
cnode7-009:635263:636084 [6] NCCL INFO NVLS multicast support is available on dev 6
cnode7-009:635263:636084 [6] NCCL INFO comm 0x555564b7b720 rank 14 nRanks 16 nNodes 2 localRanks 8 localRank 6 MNNVL 0
cnode7-009:635263:636084 [6] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 [2] 15/-1/-1->14->13 [3] 15/-1/-1->14->13 [4] 15/-1/-1->14->13 [5] 15/-1/-1->14->13 [6] 15/-1/-1->14->6 [7] -1/-1/-1->14->13 [8] 15/-1/-1->14->13 [9] 15/-1/-1->14->13 [10] 15/-1/-1->14->13 [11] 15/-1/-1->14->13 [12] 15/-1/-1->14->13 [13] 15/-1/-1->14->13 [14] 15/6/-1->14->-1 [15] -1/-1/-1->14->13
cnode7-009:635263:636084 [6] NCCL INFO P2P Chunksize set to 131072
cnode7-009:635261:636079 [4] NCCL INFO comm 0x5555694dc520 rank 12 nRanks 16 nNodes 2 localRanks 8 localRank 4 MNNVL 0
cnode7-009:635261:636079 [4] NCCL INFO Trees [0] 13/-1/-1->12->11 [1] 13/-1/-1->12->11 [2] 13/-1/-1->12->11 [3] 13/-1/-1->12->11 [4] 13/-1/-1->12->4 [5] -1/-1/-1->12->11 [6] 13/-1/-1->12->11 [7] 13/-1/-1->12->11 [8] 13/-1/-1->12->11 [9] 13/-1/-1->12->11 [10] 13/-1/-1->12->11 [11] 13/-1/-1->12->11 [12] 13/4/-1->12->-1 [13] -1/-1/-1->12->11 [14] 13/-1/-1->12->11 [15] 13/-1/-1->12->11
cnode7-009:635261:636079 [4] NCCL INFO P2P Chunksize set to 131072
cnode7-008:1504981:1505653 [4] NCCL INFO comm 0x5555694d8a50 rank 4 nRanks 16 nNodes 2 localRanks 8 localRank 4 MNNVL 0
cnode7-008:1504981:1505653 [4] NCCL INFO NVLS Head  0:  0  8
cnode7-008:1504981:1505653 [4] NCCL INFO NVLS Head  1:  1  9
cnode7-008:1504981:1505653 [4] NCCL INFO NVLS Head  2:  2 10
cnode7-008:1504981:1505653 [4] NCCL INFO NVLS Head  3:  3 11
cnode7-008:1504981:1505653 [4] NCCL INFO NVLS Head  4:  4 12
cnode7-008:1504981:1505653 [4] NCCL INFO NVLS Head  5:  5 13
cnode7-008:1504981:1505653 [4] NCCL INFO NVLS Head  6:  6 14
cnode7-008:1504981:1505653 [4] NCCL INFO NVLS Head  7:  7 15
cnode7-008:1504981:1505653 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/12/-1->4->-1 [5] -1/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->12 [13] -1/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3
cnode7-008:1504981:1505653 [4] NCCL INFO P2P Chunksize set to 131072
cnode7-009:635260:636022 [3] NCCL INFO comm 0x555564b7d7e0 rank 11 nRanks 16 nNodes 2 localRanks 8 localRank 3 MNNVL 0
cnode7-009:635260:636022 [3] NCCL INFO Trees [0] 12/-1/-1->11->10 [1] 12/-1/-1->11->10 [2] 12/-1/-1->11->10 [3] 12/-1/-1->11->3 [4] -1/-1/-1->11->10 [5] 12/-1/-1->11->10 [6] 12/-1/-1->11->10 [7] 12/-1/-1->11->10 [8] 12/-1/-1->11->10 [9] 12/-1/-1->11->10 [10] 12/-1/-1->11->10 [11] 12/3/-1->11->-1 [12] -1/-1/-1->11->10 [13] 12/-1/-1->11->10 [14] 12/-1/-1->11->10 [15] 12/-1/-1->11->10
cnode7-009:635260:636022 [3] NCCL INFO P2P Chunksize set to 131072
cnode7-009:635259:636004 [2] NCCL INFO comm 0x5555694db980 rank 10 nRanks 16 nNodes 2 localRanks 8 localRank 2 MNNVL 0
cnode7-009:635259:636004 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9 [2] 11/-1/-1->10->2 [3] -1/-1/-1->10->9 [4] 11/-1/-1->10->9 [5] 11/-1/-1->10->9 [6] 11/-1/-1->10->9 [7] 11/-1/-1->10->9 [8] 11/-1/-1->10->9 [9] 11/-1/-1->10->9 [10] 11/2/-1->10->-1 [11] -1/-1/-1->10->9 [12] 11/-1/-1->10->9 [13] 11/-1/-1->10->9 [14] 11/-1/-1->10->9 [15] 11/-1/-1->10->9
cnode7-009:635259:636004 [2] NCCL INFO P2P Chunksize set to 131072
cnode7-009:635257:636003 [0] NCCL INFO Setting affinity for GPU 0 to ff,ffffffff,ffff0000,00000000,00ffffff,ffffffff
cnode7-009:635257:636003 [0] NCCL INFO NVLS multicast support is available on dev 0
cnode7-009:635257:636003 [0] NCCL INFO comm 0x5555bdc83010 rank 8 nRanks 16 nNodes 2 localRanks 8 localRank 0 MNNVL 0
cnode7-009:635257:636003 [0] NCCL INFO Trees [0] 9/-1/-1->8->0 [1] -1/-1/-1->8->15 [2] 9/-1/-1->8->15 [3] 9/-1/-1->8->15 [4] 9/-1/-1->8->15 [5] 9/-1/-1->8->15 [6] 9/-1/-1->8->15 [7] 9/-1/-1->8->15 [8] 9/0/-1->8->-1 [9] -1/-1/-1->8->15 [10] 9/-1/-1->8->15 [11] 9/-1/-1->8->15 [12] 9/-1/-1->8->15 [13] 9/-1/-1->8->15 [14] 9/-1/-1->8->15 [15] 9/-1/-1->8->15
cnode7-009:635257:636003 [0] NCCL INFO P2P Chunksize set to 131072
cnode7-009:635258:636105 [1] NCCL INFO comm 0x5555638e63a0 rank 9 nRanks 16 nNodes 2 localRanks 8 localRank 1 MNNVL 0
cnode7-009:635258:636105 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] 10/-1/-1->9->1 [2] -1/-1/-1->9->8 [3] 10/-1/-1->9->8 [4] 10/-1/-1->9->8 [5] 10/-1/-1->9->8 [6] 10/-1/-1->9->8 [7] 10/-1/-1->9->8 [8] 10/-1/-1->9->8 [9] 10/1/-1->9->-1 [10] -1/-1/-1->9->8 [11] 10/-1/-1->9->8 [12] 10/-1/-1->9->8 [13] 10/-1/-1->9->8 [14] 10/-1/-1->9->8 [15] 10/-1/-1->9->8
cnode7-009:635258:636105 [1] NCCL INFO P2P Chunksize set to 131072
cnode7-008:1504978:1505583 [1] NCCL INFO comm 0x555562abe5e0 rank 1 nRanks 16 nNodes 2 localRanks 8 localRank 1 MNNVL 0
cnode7-008:1504978:1505583 [1] NCCL INFO NVLS Head  0:  0  8
cnode7-008:1504978:1505583 [1] NCCL INFO NVLS Head  1:  1  9
cnode7-008:1504978:1505583 [1] NCCL INFO NVLS Head  2:  2 10
cnode7-008:1504978:1505583 [1] NCCL INFO NVLS Head  3:  3 11
cnode7-008:1504978:1505583 [1] NCCL INFO NVLS Head  4:  4 12
cnode7-008:1504978:1505583 [1] NCCL INFO NVLS Head  5:  5 13
cnode7-008:1504978:1505583 [1] NCCL INFO NVLS Head  6:  6 14
cnode7-008:1504978:1505583 [1] NCCL INFO NVLS Head  7:  7 15
cnode7-008:1504978:1505583 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/9/-1->1->-1 [2] -1/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->9 [10] -1/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0
cnode7-008:1504978:1505583 [1] NCCL INFO P2P Chunksize set to 131072
cnode7-008:1504979:1505667 [2] NCCL INFO comm 0x555563055650 rank 2 nRanks 16 nNodes 2 localRanks 8 localRank 2 MNNVL 0
cnode7-008:1504979:1505667 [2] NCCL INFO NVLS Head  0:  0  8
cnode7-008:1504979:1505667 [2] NCCL INFO NVLS Head  1:  1  9
cnode7-008:1504979:1505667 [2] NCCL INFO NVLS Head  2:  2 10
cnode7-008:1504979:1505667 [2] NCCL INFO NVLS Head  3:  3 11
cnode7-008:1504979:1505667 [2] NCCL INFO NVLS Head  4:  4 12
cnode7-008:1504979:1505667 [2] NCCL INFO NVLS Head  5:  5 13
cnode7-008:1504979:1505667 [2] NCCL INFO NVLS Head  6:  6 14
cnode7-008:1504979:1505667 [2] NCCL INFO NVLS Head  7:  7 15
cnode7-008:1504979:1505667 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/10/-1->2->-1 [3] -1/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->10 [11] -1/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1
cnode7-008:1504979:1505667 [2] NCCL INFO P2P Chunksize set to 131072
cnode7-008:1504983:1505585 [6] NCCL INFO comm 0x555562abd3b0 rank 6 nRanks 16 nNodes 2 localRanks 8 localRank 6 MNNVL 0
cnode7-008:1504983:1505585 [6] NCCL INFO NVLS Head  0:  0  8
cnode7-008:1504983:1505585 [6] NCCL INFO NVLS Head  1:  1  9
cnode7-008:1504983:1505585 [6] NCCL INFO NVLS Head  2:  2 10
cnode7-008:1504983:1505585 [6] NCCL INFO NVLS Head  3:  3 11
cnode7-008:1504983:1505585 [6] NCCL INFO NVLS Head  4:  4 12
cnode7-008:1504983:1505585 [6] NCCL INFO NVLS Head  5:  5 13
cnode7-008:1504983:1505585 [6] NCCL INFO NVLS Head  6:  6 14
cnode7-008:1504983:1505585 [6] NCCL INFO NVLS Head  7:  7 15
cnode7-008:1504983:1505585 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/14/-1->6->-1 [7] -1/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->14 [15] -1/-1/-1->6->5
cnode7-008:1504983:1505585 [6] NCCL INFO P2P Chunksize set to 131072
cnode7-008:1504982:1505587 [5] NCCL INFO comm 0x555564b79df0 rank 5 nRanks 16 nNodes 2 localRanks 8 localRank 5 MNNVL 0
cnode7-008:1504982:1505587 [5] NCCL INFO NVLS Head  0:  0  8
cnode7-008:1504982:1505587 [5] NCCL INFO NVLS Head  1:  1  9
cnode7-008:1504982:1505587 [5] NCCL INFO NVLS Head  2:  2 10
cnode7-008:1504982:1505587 [5] NCCL INFO NVLS Head  3:  3 11
cnode7-008:1504982:1505587 [5] NCCL INFO NVLS Head  4:  4 12
cnode7-008:1504982:1505587 [5] NCCL INFO NVLS Head  5:  5 13
cnode7-008:1504982:1505587 [5] NCCL INFO NVLS Head  6:  6 14
cnode7-008:1504982:1505587 [5] NCCL INFO NVLS Head  7:  7 15
cnode7-008:1504982:1505587 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/13/-1->5->-1 [6] -1/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->13 [14] -1/-1/-1->5->4 [15] 6/-1/-1->5->4
cnode7-008:1504982:1505587 [5] NCCL INFO P2P Chunksize set to 131072
cnode7-008:1504984:1505584 [7] NCCL INFO comm 0x555572152360 rank 7 nRanks 16 nNodes 2 localRanks 8 localRank 7 MNNVL 0
cnode7-008:1504984:1505584 [7] NCCL INFO NVLS Head  0:  0  8
cnode7-008:1504984:1505584 [7] NCCL INFO NVLS Head  1:  1  9
cnode7-008:1504984:1505584 [7] NCCL INFO NVLS Head  2:  2 10
cnode7-008:1504984:1505584 [7] NCCL INFO NVLS Head  3:  3 11
cnode7-008:1504984:1505584 [7] NCCL INFO NVLS Head  4:  4 12
cnode7-008:1504984:1505584 [7] NCCL INFO NVLS Head  5:  5 13
cnode7-008:1504984:1505584 [7] NCCL INFO NVLS Head  6:  6 14
cnode7-008:1504984:1505584 [7] NCCL INFO NVLS Head  7:  7 15
cnode7-008:1504984:1505584 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] 0/-1/-1->7->6 [2] 0/-1/-1->7->6 [3] 0/-1/-1->7->6 [4] 0/-1/-1->7->6 [5] 0/-1/-1->7->6 [6] 0/-1/-1->7->6 [7] 0/15/-1->7->-1 [8] -1/-1/-1->7->6 [9] 0/-1/-1->7->6 [10] 0/-1/-1->7->6 [11] 0/-1/-1->7->6 [12] 0/-1/-1->7->6 [13] 0/-1/-1->7->6 [14] 0/-1/-1->7->6 [15] 0/-1/-1->7->15
cnode7-008:1504984:1505584 [7] NCCL INFO P2P Chunksize set to 131072
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 02/0 : 10[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 10/0 : 10[2] -> 2[2] [receive] via NET/IB/2/GDRDMA
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 03/0 : 2[2] -> 10[2] [send] via NET/IB/2/GDRDMA
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 11/0 : 2[2] -> 10[2] [send] via NET/IB/2/GDRDMA
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 01/0 : 9[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 09/0 : 9[1] -> 1[1] [receive] via NET/IB/1/GDRDMA
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 00/0 : 1[1] -> 9[1] [send] via NET/IB/1/GDRDMA
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 08/0 : 1[1] -> 9[1] [send] via NET/IB/1/GDRDMA
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 06/0 : 14[6] -> 6[6] [receive] via NET/IB/7/GDRDMA
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 14/0 : 14[6] -> 6[6] [receive] via NET/IB/7/GDRDMA
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 07/0 : 6[6] -> 14[6] [send] via NET/IB/7/GDRDMA
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 15/0 : 6[6] -> 14[6] [send] via NET/IB/7/GDRDMA
cnode7-009:635259:636004 [2] NCCL INFO Channel 00/0 : 10[2] -> 11[3] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 04/0 : 10[2] -> 11[3] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 06/0 : 10[2] -> 11[3] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 08/0 : 10[2] -> 11[3] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 12/0 : 10[2] -> 11[3] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 14/0 : 10[2] -> 11[3] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 00/0 : 13[5] -> 14[6] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 02/0 : 13[5] -> 14[6] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 04/0 : 13[5] -> 14[6] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 06/0 : 13[5] -> 14[6] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 08/0 : 13[5] -> 14[6] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 10/0 : 13[5] -> 14[6] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 12/0 : 13[5] -> 14[6] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 14/0 : 13[5] -> 14[6] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 04/0 : 5[5] -> 13[5] [receive] via NET/IB/6/GDRDMA
cnode7-009:635262:636000 [5] NCCL INFO Channel 12/0 : 5[5] -> 13[5] [receive] via NET/IB/6/GDRDMA
cnode7-009:635262:636000 [5] NCCL INFO Channel 05/0 : 13[5] -> 5[5] [send] via NET/IB/6/GDRDMA
cnode7-009:635262:636000 [5] NCCL INFO Channel 13/0 : 13[5] -> 5[5] [send] via NET/IB/6/GDRDMA
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 04/0 : 12[4] -> 4[4] [receive] via NET/IB/4/GDRDMA
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 12/0 : 12[4] -> 4[4] [receive] via NET/IB/4/GDRDMA
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 05/0 : 4[4] -> 12[4] [send] via NET/IB/4/GDRDMA
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 13/0 : 4[4] -> 12[4] [send] via NET/IB/4/GDRDMA
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 00/0 : 4[4] -> 3[3] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 02/0 : 4[4] -> 3[3] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 04/0 : 4[4] -> 3[3] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 06/0 : 4[4] -> 3[3] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 08/0 : 4[4] -> 3[3] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 10/0 : 4[4] -> 3[3] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 12/0 : 4[4] -> 3[3] via P2P/CUMEM
cnode7-008:1504981:1505653 [4] NCCL INFO Channel 14/0 : 4[4] -> 3[3] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 05/0 : 13[5] -> 5[5] [receive] via NET/IB/6/GDRDMA
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 13/0 : 13[5] -> 5[5] [receive] via NET/IB/6/GDRDMA
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 04/0 : 5[5] -> 13[5] [send] via NET/IB/6/GDRDMA
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 12/0 : 5[5] -> 13[5] [send] via NET/IB/6/GDRDMA
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 00/0 : 5[5] -> 4[4] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 02/0 : 5[5] -> 4[4] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 06/0 : 5[5] -> 4[4] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 08/0 : 5[5] -> 4[4] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 10/0 : 5[5] -> 4[4] via P2P/CUMEM
cnode7-008:1504982:1505587 [5] NCCL INFO Channel 14/0 : 5[5] -> 4[4] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 07/0 : 15[7] -> 7[7] [receive] via NET/IB/8/GDRDMA
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 15/0 : 15[7] -> 7[7] [receive] via NET/IB/8/GDRDMA
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 06/0 : 7[7] -> 15[7] [send] via NET/IB/8/GDRDMA
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 14/0 : 7[7] -> 15[7] [send] via NET/IB/8/GDRDMA
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 00/0 : 14[6] -> 15[7] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 02/0 : 14[6] -> 15[7] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 04/0 : 14[6] -> 15[7] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 08/0 : 14[6] -> 15[7] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 10/0 : 14[6] -> 15[7] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 12/0 : 14[6] -> 15[7] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 07/0 : 6[6] -> 14[6] [receive] via NET/IB/7/GDRDMA
cnode7-009:635263:636084 [6] NCCL INFO Channel 15/0 : 6[6] -> 14[6] [receive] via NET/IB/7/GDRDMA
cnode7-009:635263:636084 [6] NCCL INFO Channel 06/0 : 14[6] -> 6[6] [send] via NET/IB/7/GDRDMA
cnode7-009:635263:636084 [6] NCCL INFO Channel 14/0 : 14[6] -> 6[6] [send] via NET/IB/7/GDRDMA
cnode7-009:635263:636084 [6] NCCL INFO Channel 01/0 : 14[6] -> 13[5] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 03/0 : 14[6] -> 13[5] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 05/0 : 14[6] -> 13[5] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 07/0 : 14[6] -> 13[5] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 09/0 : 14[6] -> 13[5] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 11/0 : 14[6] -> 13[5] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 13/0 : 14[6] -> 13[5] via P2P/CUMEM
cnode7-009:635263:636084 [6] NCCL INFO Channel 15/0 : 14[6] -> 13[5] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 02/0 : 8[0] -> 9[1] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 04/0 : 8[0] -> 9[1] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 06/0 : 8[0] -> 9[1] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 10/0 : 8[0] -> 9[1] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 12/0 : 8[0] -> 9[1] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 14/0 : 8[0] -> 9[1] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 01/0 : 8[0] -> 15[7] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 03/0 : 8[0] -> 15[7] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 05/0 : 8[0] -> 15[7] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 07/0 : 8[0] -> 15[7] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 09/0 : 8[0] -> 15[7] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 11/0 : 8[0] -> 15[7] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 13/0 : 8[0] -> 15[7] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 15/0 : 8[0] -> 15[7] via P2P/CUMEM
cnode7-009:635257:636003 [0] NCCL INFO Channel 01/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
cnode7-009:635257:636003 [0] NCCL INFO Channel 09/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
cnode7-009:635257:636003 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
cnode7-009:635257:636003 [0] NCCL INFO Channel 08/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
cnode7-009:635264:636002 [7] NCCL INFO Channel 06/0 : 7[7] -> 15[7] [receive] via NET/IB/8/GDRDMA
cnode7-009:635264:636002 [7] NCCL INFO Channel 14/0 : 7[7] -> 15[7] [receive] via NET/IB/8/GDRDMA
cnode7-009:635264:636002 [7] NCCL INFO Channel 07/0 : 15[7] -> 7[7] [send] via NET/IB/8/GDRDMA
cnode7-009:635264:636002 [7] NCCL INFO Channel 15/0 : 15[7] -> 7[7] [send] via NET/IB/8/GDRDMA
cnode7-009:635264:636002 [7] NCCL INFO Channel 00/0 : 15[7] -> 8[0] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 02/0 : 15[7] -> 8[0] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 04/0 : 15[7] -> 8[0] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 06/0 : 15[7] -> 8[0] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 08/0 : 15[7] -> 8[0] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 10/0 : 15[7] -> 8[0] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 12/0 : 15[7] -> 8[0] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 14/0 : 15[7] -> 8[0] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 01/0 : 15[7] -> 14[6] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 03/0 : 15[7] -> 14[6] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 05/0 : 15[7] -> 14[6] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 09/0 : 15[7] -> 14[6] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 11/0 : 15[7] -> 14[6] via P2P/CUMEM
cnode7-009:635264:636002 [7] NCCL INFO Channel 13/0 : 15[7] -> 14[6] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 00/0 : 12[4] -> 13[5] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 02/0 : 12[4] -> 13[5] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 06/0 : 12[4] -> 13[5] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 08/0 : 12[4] -> 13[5] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 10/0 : 12[4] -> 13[5] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 14/0 : 12[4] -> 13[5] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 05/0 : 4[4] -> 12[4] [receive] via NET/IB/4/GDRDMA
cnode7-009:635261:636079 [4] NCCL INFO Channel 13/0 : 4[4] -> 12[4] [receive] via NET/IB/4/GDRDMA
cnode7-009:635261:636079 [4] NCCL INFO Channel 04/0 : 12[4] -> 4[4] [send] via NET/IB/4/GDRDMA
cnode7-009:635261:636079 [4] NCCL INFO Channel 12/0 : 12[4] -> 4[4] [send] via NET/IB/4/GDRDMA
cnode7-009:635261:636079 [4] NCCL INFO Channel 01/0 : 12[4] -> 11[3] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 03/0 : 12[4] -> 11[3] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 05/0 : 12[4] -> 11[3] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 07/0 : 12[4] -> 11[3] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 09/0 : 12[4] -> 11[3] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 11/0 : 12[4] -> 11[3] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 13/0 : 12[4] -> 11[3] via P2P/CUMEM
cnode7-009:635261:636079 [4] NCCL INFO Channel 15/0 : 12[4] -> 11[3] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 00/0 : 0[0] -> 7[7] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 02/0 : 0[0] -> 7[7] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 04/0 : 0[0] -> 7[7] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 06/0 : 0[0] -> 7[7] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 08/0 : 0[0] -> 7[7] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 10/0 : 0[0] -> 7[7] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 12/0 : 0[0] -> 7[7] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 14/0 : 0[0] -> 7[7] via P2P/CUMEM
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 08/0 : 8[0] -> 0[0] [receive] via NET/IB/0/GDRDMA
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 01/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA
cnode7-008:1504977:1505582 [0] NCCL INFO Channel 09/0 : 0[0] -> 8[0] [send] via NET/IB/0/GDRDMA
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 03/0 : 11[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 11/0 : 11[3] -> 3[3] [receive] via NET/IB/3/GDRDMA
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 02/0 : 3[3] -> 11[3] [send] via NET/IB/3/GDRDMA
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 10/0 : 3[3] -> 11[3] [send] via NET/IB/3/GDRDMA
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 00/0 : 3[3] -> 2[2] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 04/0 : 3[3] -> 2[2] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 06/0 : 3[3] -> 2[2] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 08/0 : 3[3] -> 2[2] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 12/0 : 3[3] -> 2[2] via P2P/CUMEM
cnode7-008:1504980:1505586 [3] NCCL INFO Channel 14/0 : 3[3] -> 2[2] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 00/0 : 11[3] -> 12[4] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 02/0 : 11[3] -> 12[4] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 04/0 : 11[3] -> 12[4] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 06/0 : 11[3] -> 12[4] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 08/0 : 11[3] -> 12[4] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 10/0 : 11[3] -> 12[4] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 12/0 : 11[3] -> 12[4] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 14/0 : 11[3] -> 12[4] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 02/0 : 3[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
cnode7-009:635260:636022 [3] NCCL INFO Channel 10/0 : 3[3] -> 11[3] [receive] via NET/IB/3/GDRDMA
cnode7-009:635260:636022 [3] NCCL INFO Channel 03/0 : 11[3] -> 3[3] [send] via NET/IB/3/GDRDMA
cnode7-009:635260:636022 [3] NCCL INFO Channel 11/0 : 11[3] -> 3[3] [send] via NET/IB/3/GDRDMA
cnode7-009:635260:636022 [3] NCCL INFO Channel 01/0 : 11[3] -> 10[2] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 05/0 : 11[3] -> 10[2] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 07/0 : 11[3] -> 10[2] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 09/0 : 11[3] -> 10[2] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 13/0 : 11[3] -> 10[2] via P2P/CUMEM
cnode7-009:635260:636022 [3] NCCL INFO Channel 15/0 : 11[3] -> 10[2] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 00/0 : 9[1] -> 10[2] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 02/0 : 9[1] -> 10[2] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 04/0 : 9[1] -> 10[2] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 06/0 : 9[1] -> 10[2] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 08/0 : 9[1] -> 10[2] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 10/0 : 9[1] -> 10[2] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 12/0 : 9[1] -> 10[2] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 14/0 : 9[1] -> 10[2] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 00/0 : 1[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
cnode7-009:635258:636105 [1] NCCL INFO Channel 08/0 : 1[1] -> 9[1] [receive] via NET/IB/1/GDRDMA
cnode7-009:635258:636105 [1] NCCL INFO Channel 01/0 : 9[1] -> 1[1] [send] via NET/IB/1/GDRDMA
cnode7-009:635258:636105 [1] NCCL INFO Channel 09/0 : 9[1] -> 1[1] [send] via NET/IB/1/GDRDMA
cnode7-009:635258:636105 [1] NCCL INFO Channel 03/0 : 9[1] -> 8[0] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 05/0 : 9[1] -> 8[0] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 07/0 : 9[1] -> 8[0] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 11/0 : 9[1] -> 8[0] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 13/0 : 9[1] -> 8[0] via P2P/CUMEM
cnode7-009:635258:636105 [1] NCCL INFO Channel 15/0 : 9[1] -> 8[0] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 00/0 : 2[2] -> 1[1] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 02/0 : 2[2] -> 1[1] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 04/0 : 2[2] -> 1[1] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 06/0 : 2[2] -> 1[1] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 08/0 : 2[2] -> 1[1] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 10/0 : 2[2] -> 1[1] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 12/0 : 2[2] -> 1[1] via P2P/CUMEM
cnode7-008:1504979:1505667 [2] NCCL INFO Channel 14/0 : 2[2] -> 1[1] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 02/0 : 1[1] -> 0[0] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 04/0 : 1[1] -> 0[0] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 06/0 : 1[1] -> 0[0] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 10/0 : 1[1] -> 0[0] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 12/0 : 1[1] -> 0[0] via P2P/CUMEM
cnode7-008:1504978:1505583 [1] NCCL INFO Channel 14/0 : 1[1] -> 0[0] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 00/0 : 6[6] -> 5[5] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 02/0 : 6[6] -> 5[5] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 04/0 : 6[6] -> 5[5] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 06/0 : 6[6] -> 5[5] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 08/0 : 6[6] -> 5[5] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 10/0 : 6[6] -> 5[5] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 12/0 : 6[6] -> 5[5] via P2P/CUMEM
cnode7-008:1504983:1505585 [6] NCCL INFO Channel 14/0 : 6[6] -> 5[5] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 03/0 : 2[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
cnode7-009:635259:636004 [2] NCCL INFO Channel 11/0 : 2[2] -> 10[2] [receive] via NET/IB/2/GDRDMA
cnode7-009:635259:636004 [2] NCCL INFO Channel 02/0 : 10[2] -> 2[2] [send] via NET/IB/2/GDRDMA
cnode7-009:635259:636004 [2] NCCL INFO Channel 10/0 : 10[2] -> 2[2] [send] via NET/IB/2/GDRDMA
cnode7-009:635259:636004 [2] NCCL INFO Channel 01/0 : 10[2] -> 9[1] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 03/0 : 10[2] -> 9[1] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 05/0 : 10[2] -> 9[1] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 07/0 : 10[2] -> 9[1] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 09/0 : 10[2] -> 9[1] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 11/0 : 10[2] -> 9[1] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 13/0 : 10[2] -> 9[1] via P2P/CUMEM
cnode7-009:635259:636004 [2] NCCL INFO Channel 15/0 : 10[2] -> 9[1] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 01/0 : 13[5] -> 12[4] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 03/0 : 13[5] -> 12[4] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 07/0 : 13[5] -> 12[4] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 09/0 : 13[5] -> 12[4] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 11/0 : 13[5] -> 12[4] via P2P/CUMEM
cnode7-009:635262:636000 [5] NCCL INFO Channel 15/0 : 13[5] -> 12[4] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 00/0 : 7[7] -> 6[6] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 02/0 : 7[7] -> 6[6] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 04/0 : 7[7] -> 6[6] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 08/0 : 7[7] -> 6[6] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 10/0 : 7[7] -> 6[6] via P2P/CUMEM
cnode7-008:1504984:1505584 [7] NCCL INFO Channel 12/0 : 7[7] -> 6[6] via P2P/CUMEM
srun: Job step aborted: Waiting up to 32 seconds for job step to finish.
slurmstepd: error: *** STEP 4946.0 ON cnode7-008 CANCELLED AT 2024-11-28T15:15:48 ***