Hugging Face's logo

amir0907
/
test2

Safetensors
VibeVoice
Model card Files
xet
 Community
amir0907 commited on
Commit
e56fafc
·
verified ·
1 Parent(s): 240be18

Upload model files from Kaggle

Browse files
Files changed (6) hide show
  1. config.json +119 -0
  2. export_info.json +100 -0
  3. model-00001-of-00002.safetensors +3 -0
  4. model-00002-of-00002.safetensors +3 -0
  5. model.safetensors.index.json +0 -0
  6. preprocessor_config.json +13 -0
config.json ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "acoustic_tokenizer_config": {
3
+ "causal": true,
4
+ "channels": 1,
5
+ "conv_bias": true,
6
+ "conv_norm": "none",
7
+ "corpus_normalize": 0.0,
8
+ "decoder_depths": null,
9
+ "decoder_n_filters": 32,
10
+ "decoder_ratios": [
11
+ 8,
12
+ 5,
13
+ 5,
14
+ 4,
15
+ 2,
16
+ 2
17
+ ],
18
+ "disable_last_norm": true,
19
+ "encoder_depths": "3-3-3-3-3-3-8",
20
+ "encoder_n_filters": 32,
21
+ "encoder_ratios": [
22
+ 8,
23
+ 5,
24
+ 5,
25
+ 4,
26
+ 2,
27
+ 2
28
+ ],
29
+ "fix_std": 0.5,
30
+ "layer_scale_init_value": 1e-06,
31
+ "layernorm": "RMSNorm",
32
+ "layernorm_elementwise_affine": true,
33
+ "layernorm_eps": 1e-05,
34
+ "mixer_layer": "depthwise_conv",
35
+ "model_type": "vibevoice_acoustic_tokenizer",
36
+ "pad_mode": "constant",
37
+ "std_dist_type": "gaussian",
38
+ "torch_dtype": "float32",
39
+ "vae_dim": 64,
40
+ "weight_init_value": 0.01
41
+ },
42
+ "acoustic_vae_dim": 64,
43
+ "architectures": [
44
+ "VibeVoiceForConditionalGeneration"
45
+ ],
46
+ "decoder_config": {
47
+ "attention_dropout": 0.0,
48
+ "hidden_act": "silu",
49
+ "hidden_size": 1536,
50
+ "initializer_range": 0.02,
51
+ "intermediate_size": 8960,
52
+ "max_position_embeddings": 65536,
53
+ "max_window_layers": 28,
54
+ "model_type": "qwen2",
55
+ "num_attention_heads": 12,
56
+ "num_hidden_layers": 28,
57
+ "num_key_value_heads": 2,
58
+ "rms_norm_eps": 1e-06,
59
+ "rope_scaling": null,
60
+ "rope_theta": 1000000.0,
61
+ "sliding_window": null,
62
+ "tie_word_embeddings": true,
63
+ "torch_dtype": "float32",
64
+ "use_cache": true,
65
+ "use_sliding_window": false,
66
+ "vocab_size": 151936
67
+ },
68
+ "diffusion_head_config": {
69
+ "ddpm_batch_mul": 4,
70
+ "ddpm_beta_schedule": "cosine",
71
+ "ddpm_num_inference_steps": 20,
72
+ "ddpm_num_steps": 1000,
73
+ "diffusion_type": "ddpm",
74
+ "head_ffn_ratio": 3.0,
75
+ "head_layers": 4,
76
+ "hidden_size": 1536,
77
+ "latent_size": 64,
78
+ "model_type": "vibevoice_diffusion_head",
79
+ "prediction_type": "v_prediction",
80
+ "rms_norm_eps": 1e-05,
81
+ "speech_vae_dim": 64,
82
+ "torch_dtype": "float32"
83
+ },
84
+ "model_type": "vibevoice",
85
+ "semantic_tokenizer_config": {
86
+ "causal": true,
87
+ "channels": 1,
88
+ "conv_bias": true,
89
+ "conv_norm": "none",
90
+ "corpus_normalize": 0.0,
91
+ "disable_last_norm": true,
92
+ "encoder_depths": "3-3-3-3-3-3-8",
93
+ "encoder_n_filters": 32,
94
+ "encoder_ratios": [
95
+ 8,
96
+ 5,
97
+ 5,
98
+ 4,
99
+ 2,
100
+ 2
101
+ ],
102
+ "fix_std": 0,
103
+ "layer_scale_init_value": 1e-06,
104
+ "layernorm": "RMSNorm",
105
+ "layernorm_elementwise_affine": true,
106
+ "layernorm_eps": 1e-05,
107
+ "mixer_layer": "depthwise_conv",
108
+ "model_type": "vibevoice_semantic_tokenizer",
109
+ "pad_mode": "constant",
110
+ "std_dist_type": "none",
111
+ "torch_dtype": "float32",
112
+ "vae_dim": 128,
113
+ "weight_init_value": 0.01
114
+ },
115
+ "semantic_vae_dim": 128,
116
+ "torch_dtype": "bfloat16",
117
+ "transformers_version": "4.51.3",
118
+ "use_cache": true
119
+ }
export_info.json ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_model": "microsoft/VibeVoice-1.5B",
3
+ "checkpoint_dir": "/kaggle/working/vibevoice-finetune/out_tpu/final",
4
+ "dtype": "bfloat16",
5
+ "components_merged": [
6
+ "language_model",
7
+ "prediction_head",
8
+ "acoustic_connector",
9
+ "semantic_connector"
10
+ ],
11
+ "tensors_changed_vs_base": 365,
12
+ "training_state": {
13
+ "step": 230,
14
+ "args": {
15
+ "model_name_or_path": "microsoft/VibeVoice-1.5B",
16
+ "data_dir": "/kaggle/working/vibevoice-finetune/preprocessed_tpu/train",
17
+ "eval_data_dir": null,
18
+ "output_dir": "out_tpu",
19
+ "lazy_data": false,
20
+ "max_seq_len": 0,
21
+ "max_latents": 0,
22
+ "max_segments": 0,
23
+ "buckets": null,
24
+ "auto_buckets": 0,
25
+ "per_device_batch_size": 2,
26
+ "gradient_accumulation_steps": 4,
27
+ "learning_rate": 1e-05,
28
+ "head_lr_multiplier": 2.0,
29
+ "weight_decay": 0.01,
30
+ "adam_beta1": 0.9,
31
+ "adam_beta2": 0.95,
32
+ "adam_eps": 1e-08,
33
+ "max_grad_norm": 1.0,
34
+ "no_master_weights": false,
35
+ "num_train_epochs": 2.0,
36
+ "max_steps": 0,
37
+ "warmup_ratio": 0.05,
38
+ "warmup_steps": 0,
39
+ "lr_scheduler": "cosine",
40
+ "min_lr_ratio": 0.05,
41
+ "seed": 42,
42
+ "ce_loss_weight": 1.0,
43
+ "diffusion_loss_weight": 1.5,
44
+ "ddpm_batch_mul": 4,
45
+ "ce_chunk_size": 256,
46
+ "no_semantic": false,
47
+ "no_latent_noise": false,
48
+ "ce_acoustic_supervision": "target",
49
+ "freeze_components": null,
50
+ "freeze_llm_layers": null,
51
+ "train_llm_layers": null,
52
+ "freeze_head_layers": null,
53
+ "freeze_embeddings": false,
54
+ "freeze_lm_head": false,
55
+ "freeze_regex": null,
56
+ "bf16": true,
57
+ "gradient_checkpointing": true,
58
+ "no_spmd": false,
59
+ "logging_steps": 10,
60
+ "save_steps": 300,
61
+ "save_total_limit": 3,
62
+ "eval_steps": 0,
63
+ "eval_batches": 20,
64
+ "resume_from_checkpoint": null,
65
+ "dataloader_workers": 0,
66
+ "profile_first_steps": 3
67
+ },
68
+ "shapes": {
69
+ "max_seq_len": 512,
70
+ "max_latents": 224,
71
+ "max_segments": 8,
72
+ "acoustic_dim": 64,
73
+ "semantic_dim": 128
74
+ },
75
+ "preprocess_metadata": {
76
+ "model_name_or_path": "microsoft/VibeVoice-1.5B",
77
+ "processor_name_or_path": "vibevoice/processor",
78
+ "compress_ratio": 3200,
79
+ "acoustic_dim": 64,
80
+ "semantic_dim": 128,
81
+ "fix_std": 0.5,
82
+ "latent_dtype": "float16",
83
+ "speech_scaling_factor": 0.1962890625,
84
+ "speech_bias_factor": -0.04931640625,
85
+ "scaling_source": "checkpoint",
86
+ "corpus_latent_mean": -0.1338784531259102,
87
+ "corpus_latent_var": 23.753134242682215,
88
+ "shapes": {
89
+ "max_seq_len": 512,
90
+ "max_latents": 224,
91
+ "max_segments": 8,
92
+ "acoustic_dim": 64,
93
+ "semantic_dim": 128
94
+ },
95
+ "pad_token_id": 151655,
96
+ "augment_target_silence": true,
97
+ "normalize_target_audio": false
98
+ }
99
+ }
100
+ }
model-00001-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0a5cf0011b5358c2dd476e23167886bdd7f3bd48641d74af4d5bb4e8fca21391
3
+ size 3991978364
model-00002-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a14a8d223566bf88a3112325cb368374476ccf7085f760b09ec2892f8269698b
3
+ size 1416224194
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
preprocessor_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "processor_class": "VibeVoiceProcessor",
3
+ "speech_tok_compress_ratio": 3200,
4
+ "db_normalize": true,
5
+ "audio_processor": {
6
+ "feature_extractor_type": "VibeVoiceTokenizerProcessor",
7
+ "sampling_rate": 24000,
8
+ "normalize_audio": true,
9
+ "target_dB_FS": -25,
10
+ "eps": 1e-06
11
+ },
12
+ "language_model_pretrained_name": "Qwen/Qwen2.5-7B"
13
+ }