Duplicate from CypressYang/SongBloom

Co-authored-by: CypressYang <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,3 @@
+---
+license: apache-2.0
+---

autoencoder_music_dsp1920.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10ccb6c83613781ad32e998a90597ba7eb9292911a224598da1fd53728eb4cd3
+size 674920616

songbloom_full_150s.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaeaee1dc889c8790e53064189ddcb7c66396cbfef3b15794a53e41d17d55fd2
+size 7827256017

songbloom_full_150s.yaml

@@ -0,0 +1,76 @@
+cfg_file: 
+precision: 'bf16-mixed' # ['16-mixed', 'bf16-mixed']
+min_dur: 60
+max_dur: 150
+sr: 48000
+
+pretrained_path: ${dynamic_path:???/songbloom_full_150s.pt}
+continue_checkpoint: 
+
+train_dataset:
+  lyric_processor: phoneme
+  prompt_len: 10
+
+vae:
+  vae_cfg: ${dynamic_path:???/stable_audio_1920_vae.json}
+  vae_ckpt: ${dynamic_path:???/autoencoder_music_dsp1920.ckpt}
+  sr: ${sr}
+
+model: 
+  block_size: 16 
+  latent_dim: 64
+  dim: 1536
+  num_heads: 24
+  lm_layers: 36
+  diff_layers: 12
+  num_pitch: 16384
+  time_cond_type: prepend 
+  timestep_features_dim: 256
+  diffusion_objective: rectified_flow
+  timestep_sampler: logit_normal
+  backend: llama
+  rotary_base_val: 20000
+  init_std: 0.02
+  h_dropout: 0.05 
+
+  condition_provider_cfg:
+    prompt_wav:
+      type: audio_tokenizer_wrapper
+      output_dim: ${model.dim}
+      audio_tokenizer: 
+      max_len: 250 # 25.0 * 10s
+    lyrics:
+      type: phoneme_tokenizer
+      output_dim: ${model.dim}
+      vocab_list: ${load_yaml:${dynamic_path:???/vocab_g2p.yaml}}
+      max_len: 600
+      max_sentence_per_structure: 50
+      mode: sum
+
+
+  cfg_dropout: 0.1
+  attribute_dropout:
+    text:
+      lyrics: 0.
+    wav:
+      prompt_wav: 0.1
+
+  fuser_cfg:
+    cross_attention_pos_emb: false
+    cross_attention_pos_emb_scale: 1
+    sum: []
+    prepend: [lyrics, prompt_wav] 
+    cross: []      
+    input_interpolate: []
+
+
+
+inference:
+  cfg_coef: 1.5 
+  temp: 0.9
+  diff_temp: 0.95
+  top_k: 100
+  penalty_repeat: True
+  penalty_window: 50
+  steps: 36
+  dit_cfg_type: h

stable_audio_1920_vae.json

@@ -0,0 +1,122 @@
+{
+    "model_type": "autoencoder",
+    "sample_size": 403200,
+    "sample_rate": 48000,
+    "audio_channels": 2,
+    "model": {
+        "encoder": {
+            "type": "oobleck",
+            "config": {
+                "in_channels": 2,
+                "channels": 128,
+                "c_mults": [1, 2, 4, 8, 16],
+                "strides": [2, 4, 4, 6, 10],
+                "latent_dim": 128,
+                "use_snake": true
+            }
+        },
+        "decoder": {
+            "type": "oobleck",
+            "config": {
+                "out_channels": 2,
+                "channels": 128,
+                "c_mults": [1, 2, 4, 8, 16],
+                "strides": [2, 4, 4, 6, 10],
+                "latent_dim": 64,
+                "use_snake": true,
+                "final_tanh": false
+            }
+        },
+        "bottleneck": {
+            "type": "vae"
+        },
+        "latent_dim": 64,
+        "downsampling_ratio": 1920,
+        "io_channels": 2
+    },
+    "training": {
+        "learning_rate": 1.5e-4,
+        "warmup_steps": 0,
+        "use_ema": true,
+        "optimizer_configs": {
+            "autoencoder": {
+                "optimizer": {
+                    "type": "AdamW",
+                    "config": {
+                        "betas": [0.8, 0.99],
+                        "lr": 1.5e-4,
+                        "weight_decay": 1e-3
+                    }
+                },
+                "scheduler": {
+                    "type": "InverseLR",
+                    "config": {
+                        "inv_gamma": 200000,
+                        "power": 0.5,
+                        "warmup": 0.999
+                    }
+                }
+            },
+            "discriminator": {
+                "optimizer": {
+                    "type": "AdamW",
+                    "config": {
+                        "betas": [0.8, 0.99],
+                        "lr": 3e-4,
+                        "weight_decay": 1e-3
+                    }
+                },
+                "scheduler": {
+                    "type": "InverseLR",
+                    "config": {
+                        "inv_gamma": 200000,
+                        "power": 0.5,
+                        "warmup": 0.999
+                    }
+                }
+            }
+        },
+        "loss_configs": {
+            "discriminator": {
+                "type": "encodec",
+                "config": {
+                    "filters": 64,
+                    "n_ffts": [2048, 1024, 512, 256, 128],
+                    "hop_lengths": [512, 256, 128, 64, 32],
+                    "win_lengths": [2048, 1024, 512, 256, 128]
+                },
+                "weights": {
+                    "adversarial": 0.1,
+                    "feature_matching": 5.0
+                }
+            },
+            "spectral": {
+                "type": "mrstft",
+                "config": {
+                    "fft_sizes": [2048, 1024, 512, 256, 128, 64, 32],
+                    "hop_sizes": [512, 256, 128, 64, 32, 16, 8],
+                    "win_lengths": [2048, 1024, 512, 256, 128, 64, 32],
+                    "perceptual_weighting": true
+                },
+                "weights": {
+                    "mrstft": 1.0
+                }
+            },
+            "time": {
+                "type": "l1",
+                "weights": {
+                    "l1": 0.0
+                }
+            },
+            "bottleneck": {
+                "type": "kl",
+                "weights": {
+                    "kl": 1e-4
+                }
+            }
+        },
+        "demo": {
+            "demo_every": 2000
+        }
+    }
+}

vocab_g2p.yaml

@@ -0,0 +1,330 @@
+- ''
+- '[verse]'
+- '[chorus]'
+- '[bridge]'
+- '[intro]'
+- '[outro]'
+- '[inst]'
+- '[silence]'
+- '!'
+- ','
+- '-'
+- .
+- '?'
+- AA
+- AA0
+- AA1
+- AA2
+- AE0
+- AE1
+- AE2
+- AH0
+- AH1
+- AH2
+- AO0
+- AO1
+- AO2
+- AW0
+- AW1
+- AW2
+- AY0
+- AY1
+- AY2
+- B
+- CH
+- D
+- DH
+- E1
+- E2
+- E3
+- E4
+- E5
+- EE
+- EH0
+- EH1
+- EH2
+- ER
+- ER0
+- ER1
+- ER2
+- EY0
+- EY1
+- EY2
+- En1
+- En2
+- En3
+- En4
+- En5
+- F
+- G
+- HH
+- I
+- IH
+- IH0
+- IH1
+- IH2
+- IY0
+- IY1
+- IY2
+- JH
+- K
+- L
+- M
+- N
+- NG
+- OO
+- OW0
+- OW1
+- OW2
+- OY0
+- OY1
+- OY2
+- P
+- R
+- S
+- SH
+- SP
+- SP2
+- SP3
+- T
+- TH
+- U
+- UH0
+- UH1
+- UH2
+- UNK
+- UW0
+- UW1
+- UW2
+- V
+- W
+- Y
+- Z
+- ZH
+- _
+- a
+- a1
+- a2
+- a3
+- a4
+- a5
+- ai1
+- ai2
+- ai3
+- ai4
+- ai5
+- an1
+- an2
+- an3
+- an4
+- an5
+- ang1
+- ang2
+- ang3
+- ang4
+- ang5
+- ao1
+- ao2
+- ao3
+- ao4
+- ao5
+- b
+- by
+- c
+- ch
+- cl
+- d
+- dy
+- e
+- e1
+- e2
+- e3
+- e4
+- e5
+- ei1
+- ei2
+- ei3
+- ei4
+- ei5
+- en1
+- en2
+- en3
+- en4
+- en5
+- eng1
+- eng2
+- eng3
+- eng4
+- eng5
+- er1
+- er2
+- er3
+- er4
+- er5
+- f
+- g
+- gy
+- h
+- hy
+- i
+- i01
+- i02
+- i03
+- i04
+- i05
+- i1
+- i2
+- i3
+- i4
+- i5
+- ia1
+- ia2
+- ia3
+- ia4
+- ia5
+- ian1
+- ian2
+- ian3
+- ian4
+- ian5
+- iang1
+- iang2
+- iang3
+- iang4
+- iang5
+- iao1
+- iao2
+- iao3
+- iao4
+- iao5
+- ie1
+- ie2
+- ie3
+- ie4
+- ie5
+- in1
+- in2
+- in3
+- in4
+- in5
+- ing1
+- ing2
+- ing3
+- ing4
+- ing5
+- iong1
+- iong2
+- iong3
+- iong4
+- iong5
+- ir1
+- ir2
+- ir3
+- ir4
+- ir5
+- iu1
+- iu2
+- iu3
+- iu4
+- iu5
+- j
+- k
+- ky
+- l
+- m
+- my
+- n
+- ny
+- o
+- o1
+- o2
+- o3
+- o4
+- o5
+- ong1
+- ong2
+- ong3
+- ong4
+- ong5
+- ou1
+- ou2
+- ou3
+- ou4
+- ou5
+- p
+- py
+- q
+- r
+- ry
+- s
+- sh
+- t
+- ts
+- u
+- u1
+- u2
+- u3
+- u4
+- u5
+- ua1
+- ua2
+- ua3
+- ua4
+- ua5
+- uai1
+- uai2
+- uai3
+- uai4
+- uai5
+- uan1
+- uan2
+- uan3
+- uan4
+- uan5
+- uang1
+- uang2
+- uang3
+- uang4
+- uang5
+- ui1
+- ui2
+- ui3
+- ui4
+- ui5
+- un1
+- un2
+- un3
+- un4
+- un5
+- uo1
+- uo2
+- uo3
+- uo4
+- uo5
+- v
+- v1
+- v2
+- v3
+- v4
+- v5
+- van1
+- van2
+- van3
+- van4
+- van5
+- ve1
+- ve2
+- ve3
+- ve4
+- ve5
+- vn1
+- vn2
+- vn3
+- vn4
+- vn5
+- w
+- x
+- y
+- z
+- zh
+- "\u2026"