Add fine-tuned MusicGen model

config.json

@@ -1,338 +1,57 @@
 {
-    "transformer_lm.norm_first": {
-        "value": true
-    },
-    "wandb.with_media_logging": {
-        "value": true
-    },
-    "generate.lm.prompt_duration": {
-        "value": "None"
-    },
-    "slurm.time": {
-        "value": 3600
-    },
-    "fuser.cross": {
-        "value": "['description']"
-    },
-    "fsdp.per_block": {
-        "value": true
-    },
-    "fsdp.buffer_dtype": {
-        "value": "float32"
-    },
-    "autocast": {
-        "value": true
-    },
-    "fsdp.param_dtype": {
-        "value": "float16"
-    },
-    "optim.eager_sync": {
-        "value": true
-    },
-    "transformer_lm.emb_lr": {
-        "value": "None"
-    },
-    "channels": {
-        "value": 1
-    },
-    "optim.ema.use": {
-        "value": true
-    },
-    "dataset.shuffle": {
-        "value": false
-    },
-    "generate.every": {
-        "value": 25
-    },
-    "codebooks_pattern.modeling": {
-        "value": "delay"
-    },
-    "metrics.text_consistency.clap.model_arch": {
-        "value": "HTSAT-base"
-    },
-    "generate.audio.loudness_headroom_db": {
-        "value": 14
-    },
-    "fuser.sum": {
-        "value": "[]"
-    },
-    "conditioners.description.t5.word_dropout": {
-        "value": 0.3
-    },
-    "dora.dir": {
-        "value": "/checkpoint/choihj/experiments/audiocraft/outputs"
-    },
-    "tensorboard.with_media_logging": {
-        "value": true
-    },
-    "generate.audio.format": {
-        "value": "wav"
-    },
-    "logging.level": {
-        "value": "INFO"
-    },
-    "slurm.gpus": {
-        "value": 4
-    },
-    "dataset.min_segment_ratio": {
-        "value": 0.8
-    },
-    "interleave_stereo_codebooks.use": {
-        "value": false
-    },
-    "codebooks_pattern.unroll.flattening": {
-        "value": "[0, 1, 2, 3]"
-    },
-    "transformer_lm.two_step_cfg": {
-        "value": false
-    },
-    "optim.updates_per_epoch": {
-        "value": 100
-    },
-    "transformer_lm.depthwise_init": {
-        "value": "current"
-    },
-    "transformer_lm.past_context": {
-        "value": "None"
-    },
-    "metrics.chroma_cosine.chroma_base.sample_rate": {
-        "value": 32000
-    },
-    "fuser.cross_attention_pos_emb_scale": {
-        "value": 1
-    },
-    "optim.epochs": {
-        "value": 100
-    },
-    "transformer_lm.bias_attn": {
-        "value": false
-    },
-    "datasource.valid": {
-        "value": "/content/drive/MyDrive/projects/carecruise_intern/audiocraft/egs/eval"
-    },
-    "tensorboard.sub_dir": {
-        "value": "None"
-    },
-    "generate.num_workers": {
-        "value": 5
-    },
-    "metrics.fad.tf.bin": {
-        "value": "None"
-    },
-    "fsdp.reduce_dtype": {
-        "value": "float32"
-    },
-    "dataset.train.merge_text_p": {
-        "value": 0.25
-    },
-    "schedule.step.gamma": {
-        "value": "None"
-    },
-    "transformer_lm.kv_repeat": {
-        "value": 1
-    },
-    "wandb.group": {
-        "value": "None"
-    },
-    "cache.write": {
-        "value": false
-    },
-    "transformer_lm.causal": {
-        "value": true
-    },
-    "generate.lm.remove_prompts": {
-        "value": false
-    },
-    "metrics.fad.tf.model_path": {
-        "value": "//reference/fad/vggish_model.ckpt"
-    },
-    "evaluate.metrics.base": {
-        "value": false
-    },
-    "generate.num_samples": {
-        "value": 5
-    },
-    "autocast_dtype": {
-        "value": "float16"
-    },
-    "classifier_free_guidance.inference_coef": {
-        "value": 3
-    },
-    "codebooks_pattern.delay.flatten_first": {
-        "value": 0
-    },
-    "dataset.segment_duration": {
-        "value": 30
-    },
-    "slurm.mem_per_gpu": {
-        "value": 40
-    },
-    "datasource.train": {
-        "value": "/content/drive/MyDrive/projects/carecruise_intern/audiocraft/egs/train"
-    },
-    "transformer_lm.layer_scale": {
-        "value": "None"
-    },
-    "num_threads": {
-        "value": 1
-    },
-    "optim.ema.device": {
-        "value": "cuda"
-    },
-    "metrics.text_consistency.use_gt": {
-        "value": false
-    },
-    "schedule.inverse_sqrt.warmup_init_lr": {
-        "value": 0
-    },
-    "evaluate.metrics.text_consistency": {
-        "value": false
-    },
-    "schedule.polynomial_decay.end_lr": {
-        "value": 0
-    },
-    "transformer_lm.num_heads": {
-        "value": 16
-    },
-    "metrics.chroma_cosine.chroma_base.n_chroma": {
-        "value": 12
-    },
-    "dtype": {
-        "value": "float32"
-    },
-    "metrics.kld.model": {
-        "value": "passt"
-    },
-    "evaluate.truncate_audio": {
-        "value": "None"
-    },
-    "checkpoint.save_last": {
-        "value": true
-    },
-    "evaluate.metrics.kld": {
-        "value": false
-    },
-    "optim.optimizer": {
-        "value": "adamw"
-    },
-    "dataset.train.drop_other_p": {
-        "value": 0.5
-    },
-    "transformer_lm.activation": {
-        "value": "gelu"
-    },
-    "evaluate.every": {
-        "value": 25
-    },
-    "fsdp.use": {
-        "value": false
-    },
-    "tokens.padding_with_special_token": {
-        "value": false
-    },
-    "transformer_lm.qk_layer_norm": {
-        "value": false
-    },
-    "device": {
-        "value": "cuda"
-    },
-    "fsdp.sharding_strategy": {
-        "value": "shard_grad_op"
-    },
-    "dataset.train.shuffle": {
-        "value": true
-    },
-    "optim.adam.betas": {
-        "value": "[0.9, 0.95]"
-    },
-    "metrics.kld.use_gt": {
-        "value": false
-    },
-    "dataset.generate.return_info": {
-        "value": true
-    },
-    "dataset.batch_size": {
-        "value": 1
-    },
-    "dataset.sample_on_duration": {
-        "value": false
-    },
-    "schedule.inverse_sqrt.warmup": {
-        "value": "None"
-    },
-    "fuser.prepend": {
-        "value": "[]"
-    },
-    "efficient_attention_backend": {
-        "value": "torch"
-    },
-    "codebooks_pattern.unroll.delays": {
-        "value": "[0, 0, 0, 0]"
-    },
-    "schedule.cosine.warmup": {
-        "value": 8
-    },
-    "schedule.lr_scheduler": {
-        "value": "cosine"
-    },
-    "dataset.valid.num_samples": {
-        "value": 1
-    },
-    "transformer_lm.hidden_scale": {
-        "value": 4
-    },
-    "schedule.exponential.lr_decay": {
-        "value": "None"
-    },
-    "show": {
-        "value": false
-    },
-    "transformer_lm.card": {
-        "value": 2048
-    },
-    "fuser.cross_attention_pos_emb": {
-        "value": false
-    },
-    "conditioners.description.model": {
-        "value": "t5"
-    },
-    "generate.path": {
-        "value": "samples"
-    },
-    "codebooks_pattern.delay.delays": {
-        "value": "[0, 1, 2, 3]"
-    },
-    "transformer_lm.xpos": {
-        "value": false
-    },
-    "logging.log_tensorboard": {
-        "value": true
-    },
-    "benchmark_no_load": {
-        "value": false
-    },
-    "schedule.cosine.lr_min_ratio": {
-        "value": 0
-    },
-    "transformer_lm.custom": {
-        "value": false
-    },
-    "evaluate.metrics.chroma_cosine": {
-        "value": false
-    },
-    "cache.write_shard": {
-        "value": 0
-    },
-    "schedule.polynomial_decay.power": {
-        "value": 1
-    },
-    "generate.audio.strategy": {
-        "value": "loudness"
-    },
-    "transformer_lm.dim": {
-        "value": 1024
-    },
-    "compression_model_checkpoint": {
-        "value": "//pretrained/facebook/encodec_32khz"
+    "model_type": "musicgen",
+    "text_encoder": {
+        "type": "T5",
+        "name_or_path": "t5-base",
+        "config": {
+            "vocab_size": 32128,
+            "d_model": 1024,
+            "num_layers": 12,
+            "num_heads": 16,
+            "dropout_rate": 0.1
+        }
+    },
+    "audio_encoder": {
+        "type": "AudioEncoder",
+        "config": {
+            "sample_rate": 32000,
+            "num_channels": 1,
+            "embedding_size": 512
+        }
+    },
+    "decoder": {
+        "type": "TransformerDecoder",
+        "config": {
+            "d_model": 1024,
+            "num_heads": 16,
+            "num_layers": 24,
+            "dropout_rate": 0.1
+        }
+    },
+    "training": {
+        "batch_size": 16,
+        "num_epochs": 100,
+        "learning_rate": 0.0001,
+        "weight_decay": 0.01,
+        "gradient_clipping": 1.0
+    },
+    "generation": {
+        "sample_rate": 32000,
+        "audio_format": "wav",
+        "num_samples": 5,
+        "max_duration": 30.0,
+        "temperature": 1.0,
+        "top_k": 250,
+        "top_p": 0.9
+    },
+    "logging": {
+        "log_tensorboard": true,
+        "log_wandb": true,
+        "wandb_project": "music_generation",
+        "log_updates": 10
+    },
+    "hardware": {
+        "device": "cuda",
+        "num_gpus": 4
     }
 }