Update model

README.md

@@ -0,0 +1,292 @@
+---
+tags:
+- espnet
+- audio
+- universa
+language: multilingual
+datasets:
+- universa_unite
+license: cc-by-4.0
+---
+
+## ESPnet2 universa model
+
+### `espnet/arecho_base_v0.1-large-decoder`
+
+This model was trained by ftshijt using universa_unite recipe in [espnet](https://github.com/espnet/espnet/).
+
+### Demo: How to use in ESPnet2
+
+Follow the [ESPnet installation instructions](https://espnet.github.io/espnet/installation.html)
+if you haven't done that already.
+
+```bash
+cd espnet
+git checkout 69996dc206e556ec48db77b6cc385ff1d32895b3
+pip install -e .
+cd egs2/universa_unite/uni_versa1
+./run.sh --skip_data_prep false --skip_train true --download_model espnet/arecho_base_v0.1-large-decoder
+```
+
+
+
+## universa config
+
+<details><summary>expand</summary>
+
+```
+config: conf/train_aruniversa_wavlm_large.yaml
+print_config: false
+log_level: INFO
+drop_last_iter: false
+dry_run: false
+iterator_type: sequence
+valid_iterator_type: null
+output_dir: exp/universa_universa_ar_overall_base_token_wavlm_large
+ngpu: 1
+seed: 777
+num_workers: 1
+num_att_plot: 0
+dist_backend: nccl
+dist_init_method: env://
+dist_world_size: null
+dist_rank: null
+local_rank: 0
+dist_master_addr: null
+dist_master_port: null
+dist_launcher: null
+multiprocessing_distributed: false
+unused_parameters: false
+sharded_ddp: false
+use_deepspeed: false
+deepspeed_config: null
+gradient_as_bucket_view: true
+ddp_comm_hook: null
+cudnn_enabled: true
+cudnn_benchmark: false
+cudnn_deterministic: false
+use_tf32: false
+collect_stats: false
+write_collected_feats: false
+max_epoch: 100
+patience: null
+val_scheduler_criterion:
+- valid
+- loss
+early_stopping_criterion:
+- valid
+- loss
+- min
+best_model_criterion:
+-   - train
+    - loss
+    - min
+-   - valid
+    - loss
+    - min
+-   - train
+    - acc
+    - max
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 1
+nbest_averaging_interval: 0
+grad_clip: -1
+grad_clip_type: 2.0
+grad_noise: false
+accum_grad: 2
+no_forward_run: false
+resume: true
+train_dtype: float32
+use_amp: false
+log_interval: 50
+use_matplotlib: true
+use_tensorboard: true
+create_graph_in_tensorboard: false
+use_wandb: false
+wandb_project: null
+wandb_id: null
+wandb_entity: null
+wandb_name: null
+wandb_model_log_interval: -1
+detect_anomaly: false
+use_adapter: false
+adapter: lora
+save_strategy: all
+adapter_conf: {}
+pretrain_path: null
+init_param: []
+ignore_init_mismatch: false
+freeze_param:
+- frontend.upstream
+num_iters_per_epoch: null
+batch_size: 16
+valid_batch_size: null
+batch_bins: 1000000
+valid_batch_bins: null
+category_sample_size: 10
+train_shape_file:
+- exp/universa_stats_overall_base/train/audio_shape
+- exp/universa_stats_overall_base/train/ref_audio_shape
+valid_shape_file:
+- exp/universa_stats_overall_base/valid/audio_shape
+- exp/universa_stats_overall_base/valid/ref_audio_shape
+batch_type: sorted
+valid_batch_type: null
+fold_length:
+- 256000
+sort_in_batch: descending
+shuffle_within_batch: false
+sort_batch: descending
+multiple_iterator: false
+chunk_length: 500
+chunk_shift_ratio: 0.5
+num_cache_chunks: 1024
+chunk_excluded_key_prefixes: []
+chunk_default_fs: null
+chunk_max_abs_length: null
+chunk_discard_short_samples: true
+train_data_path_and_name_and_type:
+-   - dump/raw/overall_base/wav.scp
+    - audio
+    - kaldi_ark
+-   - dump/raw/overall_base/metric.scp
+    - metrics
+    - metric
+-   - dump/raw/overall_base/ref_wav.scp
+    - ref_audio
+    - kaldi_ark
+valid_data_path_and_name_and_type:
+-   - dump/raw/overall_dev/wav.scp
+    - audio
+    - kaldi_ark
+-   - dump/raw/overall_dev/metric.scp
+    - metrics
+    - metric
+-   - dump/raw/overall_dev/ref_wav.scp
+    - ref_audio
+    - kaldi_ark
+multi_task_dataset: false
+allow_variable_data_keys: false
+max_cache_size: 0.0
+max_cache_fd: 32
+allow_multi_rates: false
+valid_max_cache_size: null
+exclude_weight_decay: false
+exclude_weight_decay_conf: {}
+optim: adamw
+optim_conf:
+    lr: 0.001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 25000
+metric2id: dump/raw/overall_base/metric2id
+metric2type: dump/raw/overall_base/metric2type
+metric_pad_value: -100
+token_list: null
+metric_token_info: data/token_list/metric_500_percentile_overall_base_w-numerical/tokens.json
+metric_token_pad_value: 0
+tokenize_numerical_metric: true
+init: null
+model_conf: {}
+use_ref_audio: true
+use_ref_text: false
+use_preprocessor: true
+token_type: bpe
+bpemodel: null
+non_linguistic_symbols: null
+cleaner: null
+g2p: null
+sequential_metric: true
+randomize_sequential_metric: true
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: wavlm_large
+    download_dir: ./hub
+    multilayer_feature: true
+universa: ar_universa
+universa_conf:
+    embedding_dim: 512
+    audio_encoder_type: transformer
+    audio_encoder_params:
+        num_blocks: 4
+        attention_heads: 4
+        linear_units: 1024
+        dropout_rate: 0.1
+        positional_dropout_rate: 0.1
+        attention_dropout_rate: 0.1
+        input_layer: conv2d
+        normalize_before: true
+        concat_after: false
+        positionwise_layer_type: linear
+        positionwise_conv_kernel_size: 1
+        layer_drop_rate: 0.1
+        qk_norm: false
+        use_flash_attn: false
+    cross_attention_type: multihead
+    cross_attention_params:
+        n_head: 2
+        dropout_rate: 0.1
+    metric_decoder_params:
+        num_blocks: 12
+        attention_heads: 8
+        linear_units: 2048
+        dropout_rate: 0.1
+        positional_dropout_rate: 0.1
+        src_attention_dropout_rate: 0.1
+        self_attention_dropout_rate: 0.1
+        input_layer: embed
+        normalize_before: true
+        concat_after: false
+        layer_drop_rate: 0.1
+        qk_norm: false
+        use_flash_attn: false
+    use_rope: true
+    lsm_weight: 0.1
+    sym_sos: <sos>
+    sym_eos: <eos>
+required:
+- output_dir
+- metric2id
+version: '202503'
+distributed: false
+```
+
+</details>
+
+
+
+### Citing ESPnet
+
+```BibTex
+@inproceedings{watanabe2018espnet,
+  author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai},
+  title={{ESPnet}: End-to-End Speech Processing Toolkit},
+  year={2018},
+  booktitle={Proceedings of Interspeech},
+  pages={2207--2211},
+  doi={10.21437/Interspeech.2018-1456},
+  url={http://dx.doi.org/10.21437/Interspeech.2018-1456}
+}
+
+
+
+
+
+
+```
+
+or arXiv:
+
+```bibtex
+@misc{watanabe2018espnet,
+  title={ESPnet: End-to-End Speech Processing Toolkit},
+  author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai},
+  year={2018},
+  eprint={1804.00015},
+  archivePrefix={arXiv},
+  primaryClass={cs.CL}
+}
+```

dump/raw/overall_base/metric2id

@@ -0,0 +1,87 @@
+srmr
+language
+nisqa_mos_pred
+nisqa_noi_pred
+nisqa_dis_pred
+nisqa_col_pred
+nisqa_loud_pred
+sheet_ssqa
+utmos
+utmosv2
+dns_overall
+dns_p808
+plcmos
+singmos
+scoreq_nr
+se_sdr
+se_sar
+se_si_snr
+se_ci_sdr
+pam_score
+speaking_rate
+audiobox_aesthetics_CE
+audiobox_aesthetics_CU
+audiobox_aesthetics_PC
+audiobox_aesthetics_PQ
+asvspoof_score
+real_language
+qwen_speaker_count
+qwen_speaker_gender
+qwen_speaker_age
+qwen_speech_impairment
+qwen_voice_pitch
+qwen_pitch_range
+qwen_voice_type
+qwen_speech_volume_level
+qwen_language
+qwen_speech_register
+qwen_vocabulary_complexity
+qwen_speech_purpose
+qwen_speech_emotion
+qwen_speech_clarity
+qwen_speech_rate
+qwen_speaking_style
+qwen_laughter_crying
+qwen_speech_background_environment
+qwen_recording_quality
+qwen_channel_type
+snr_simulation
+rir_room_size
+nomad
+emotion_similarity
+noresqa_score
+speech_bert
+speech_bleu
+speech_token_distance
+scoreq_ref
+asr_match_error_rate
+ref_text_length
+pred_text_length
+spk_similarity
+rt60
+visqol
+pysepm_fwsegsnr
+pysepm_llr
+pysepm_wss
+pysepm_cd
+pysepm_c_sig
+pysepm_c_bak
+pysepm_c_ovl
+pysepm_csii_high
+pysepm_csii_mid
+pysepm_csii_low
+pysepm_ncm
+mcd
+f0rmse
+f0corr
+pesq
+stoi
+sdr
+sar
+si_snr
+ci_sdr
+nisqa_real_mos
+wer
+cer
+urgent_mos
+voicemos_real_mos

dump/raw/overall_base/metric2type

@@ -0,0 +1,87 @@
+srmr numerical
+language categorical
+nisqa_mos_pred numerical
+nisqa_noi_pred numerical
+nisqa_dis_pred numerical
+nisqa_col_pred numerical
+nisqa_loud_pred numerical
+sheet_ssqa numerical
+utmos numerical
+utmosv2 numerical
+dns_overall numerical
+dns_p808 numerical
+plcmos numerical
+singmos numerical
+scoreq_nr numerical
+se_sdr numerical
+se_sar numerical
+se_si_snr numerical
+se_ci_sdr numerical
+pam_score numerical
+speaking_rate numerical
+audiobox_aesthetics_CE numerical
+audiobox_aesthetics_CU numerical
+audiobox_aesthetics_PC numerical
+audiobox_aesthetics_PQ numerical
+asvspoof_score numerical
+real_language categorical
+qwen_speaker_count numerical
+qwen_speaker_gender categorical
+qwen_speaker_age categorical
+qwen_speech_impairment categorical
+qwen_voice_pitch categorical
+qwen_pitch_range categorical
+qwen_voice_type categorical
+qwen_speech_volume_level categorical
+qwen_language categorical
+qwen_speech_register categorical
+qwen_vocabulary_complexity categorical
+qwen_speech_purpose categorical
+qwen_speech_emotion categorical
+qwen_speech_clarity categorical
+qwen_speech_rate categorical
+qwen_speaking_style categorical
+qwen_laughter_crying categorical
+qwen_speech_background_environment categorical
+qwen_recording_quality categorical
+qwen_channel_type categorical
+snr_simulation numerical
+rir_room_size categorical
+nomad numerical
+emotion_similarity numerical
+noresqa_score numerical
+speech_bert numerical
+speech_bleu numerical
+speech_token_distance numerical
+scoreq_ref numerical
+asr_match_error_rate numerical
+ref_text_length numerical
+pred_text_length numerical
+spk_similarity numerical
+rt60 numerical
+visqol numerical
+pysepm_fwsegsnr numerical
+pysepm_llr numerical
+pysepm_wss numerical
+pysepm_cd numerical
+pysepm_c_sig numerical
+pysepm_c_bak numerical
+pysepm_c_ovl numerical
+pysepm_csii_high numerical
+pysepm_csii_mid numerical
+pysepm_csii_low numerical
+pysepm_ncm numerical
+mcd numerical
+f0rmse numerical
+f0corr numerical
+pesq numerical
+stoi numerical
+sdr numerical
+sar numerical
+si_snr numerical
+ci_sdr numerical
+nisqa_real_mos numerical
+wer numerical
+cer numerical
+urgent_mos numerical
+voicemos_real_mos numerical

exp/universa_universa_ar_overall_base_token_wavlm_large/24epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:011b048aa2986a73de04ab5859d71a3ff822e73bbb7a357f082a9872367b8bf6
+size 2762035766

exp/universa_universa_ar_overall_base_token_wavlm_large/config.yaml

@@ -0,0 +1,217 @@
+config: conf/train_aruniversa_wavlm_large.yaml
+print_config: false
+log_level: INFO
+drop_last_iter: false
+dry_run: false
+iterator_type: sequence
+valid_iterator_type: null
+output_dir: exp/universa_universa_ar_overall_base_token_wavlm_large
+ngpu: 1
+seed: 777
+num_workers: 1
+num_att_plot: 0
+dist_backend: nccl
+dist_init_method: env://
+dist_world_size: null
+dist_rank: null
+local_rank: 0
+dist_master_addr: null
+dist_master_port: null
+dist_launcher: null
+multiprocessing_distributed: false
+unused_parameters: false
+sharded_ddp: false
+use_deepspeed: false
+deepspeed_config: null
+gradient_as_bucket_view: true
+ddp_comm_hook: null
+cudnn_enabled: true
+cudnn_benchmark: false
+cudnn_deterministic: false
+use_tf32: false
+collect_stats: false
+write_collected_feats: false
+max_epoch: 100
+patience: null
+val_scheduler_criterion:
+- valid
+- loss
+early_stopping_criterion:
+- valid
+- loss
+- min
+best_model_criterion:
+-   - train
+    - loss
+    - min
+-   - valid
+    - loss
+    - min
+-   - train
+    - acc
+    - max
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 1
+nbest_averaging_interval: 0
+grad_clip: -1
+grad_clip_type: 2.0
+grad_noise: false
+accum_grad: 2
+no_forward_run: false
+resume: true
+train_dtype: float32
+use_amp: false
+log_interval: 50
+use_matplotlib: true
+use_tensorboard: true
+create_graph_in_tensorboard: false
+use_wandb: false
+wandb_project: null
+wandb_id: null
+wandb_entity: null
+wandb_name: null
+wandb_model_log_interval: -1
+detect_anomaly: false
+use_adapter: false
+adapter: lora
+save_strategy: all
+adapter_conf: {}
+pretrain_path: null
+init_param: []
+ignore_init_mismatch: false
+freeze_param:
+- frontend.upstream
+num_iters_per_epoch: null
+batch_size: 16
+valid_batch_size: null
+batch_bins: 1000000
+valid_batch_bins: null
+category_sample_size: 10
+train_shape_file:
+- exp/universa_stats_overall_base/train/audio_shape
+- exp/universa_stats_overall_base/train/ref_audio_shape
+valid_shape_file:
+- exp/universa_stats_overall_base/valid/audio_shape
+- exp/universa_stats_overall_base/valid/ref_audio_shape
+batch_type: sorted
+valid_batch_type: null
+fold_length:
+- 256000
+sort_in_batch: descending
+shuffle_within_batch: false
+sort_batch: descending
+multiple_iterator: false
+chunk_length: 500
+chunk_shift_ratio: 0.5
+num_cache_chunks: 1024
+chunk_excluded_key_prefixes: []
+chunk_default_fs: null
+chunk_max_abs_length: null
+chunk_discard_short_samples: true
+train_data_path_and_name_and_type:
+-   - dump/raw/overall_base/wav.scp
+    - audio
+    - kaldi_ark
+-   - dump/raw/overall_base/metric.scp
+    - metrics
+    - metric
+-   - dump/raw/overall_base/ref_wav.scp
+    - ref_audio
+    - kaldi_ark
+valid_data_path_and_name_and_type:
+-   - dump/raw/overall_dev/wav.scp
+    - audio
+    - kaldi_ark
+-   - dump/raw/overall_dev/metric.scp
+    - metrics
+    - metric
+-   - dump/raw/overall_dev/ref_wav.scp
+    - ref_audio
+    - kaldi_ark
+multi_task_dataset: false
+allow_variable_data_keys: false
+max_cache_size: 0.0
+max_cache_fd: 32
+allow_multi_rates: false
+valid_max_cache_size: null
+exclude_weight_decay: false
+exclude_weight_decay_conf: {}
+optim: adamw
+optim_conf:
+    lr: 0.001
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 25000
+metric2id: dump/raw/overall_base/metric2id
+metric2type: dump/raw/overall_base/metric2type
+metric_pad_value: -100
+token_list: null
+metric_token_info: data/token_list/metric_500_percentile_overall_base_w-numerical/tokens.json
+metric_token_pad_value: 0
+tokenize_numerical_metric: true
+init: null
+model_conf: {}
+use_ref_audio: true
+use_ref_text: false
+use_preprocessor: true
+token_type: bpe
+bpemodel: null
+non_linguistic_symbols: null
+cleaner: null
+g2p: null
+sequential_metric: true
+randomize_sequential_metric: true
+frontend: s3prl
+frontend_conf:
+    frontend_conf:
+        upstream: wavlm_large
+    download_dir: ./hub
+    multilayer_feature: true
+universa: ar_universa
+universa_conf:
+    embedding_dim: 512
+    audio_encoder_type: transformer
+    audio_encoder_params:
+        num_blocks: 4
+        attention_heads: 4
+        linear_units: 1024
+        dropout_rate: 0.1
+        positional_dropout_rate: 0.1
+        attention_dropout_rate: 0.1
+        input_layer: conv2d
+        normalize_before: true
+        concat_after: false
+        positionwise_layer_type: linear
+        positionwise_conv_kernel_size: 1
+        layer_drop_rate: 0.1
+        qk_norm: false
+        use_flash_attn: false
+    cross_attention_type: multihead
+    cross_attention_params:
+        n_head: 2
+        dropout_rate: 0.1
+    metric_decoder_params:
+        num_blocks: 12
+        attention_heads: 8
+        linear_units: 2048
+        dropout_rate: 0.1
+        positional_dropout_rate: 0.1
+        src_attention_dropout_rate: 0.1
+        self_attention_dropout_rate: 0.1
+        input_layer: embed
+        normalize_before: true
+        concat_after: false
+        layer_drop_rate: 0.1
+        qk_norm: false
+        use_flash_attn: false
+    use_rope: true
+    lsm_weight: 0.1
+    sym_sos: <sos>
+    sym_eos: <eos>
+required:
+- output_dir
+- metric2id
+version: '202503'
+distributed: false

meta.yaml

@@ -0,0 +1,8 @@
+espnet: '202503'
+files:
+  model_file: exp/universa_universa_ar_overall_base_token_wavlm_large/24epoch.pth
+python: "3.9.18 | packaged by conda-forge | (main, Dec 23 2023, 17:20:25) \n[GCC 12.3.0]"
+timestamp: 1749800668.161494
+torch: 2.6.0.dev20241210+cu124
+yaml_files:
+  train_config: exp/universa_universa_ar_overall_base_token_wavlm_large/config.yaml