jax_model/hyperparams.json

{
    "dataset_name": "GCA_000001405.15_GRCh38_genomic_no_mito_upper_N",
    "alphabet": "k-mers",
    "k_for_kmers": 6,
    "num_warmup_updates": 16000,
    "warmup_init_lr": 5e-05,
    "warmup_end_lr": 0.0001,
    "training_set_proportion": 0.95,
    "tokens_per_batch": 1000000.0,
    "masking_ratio": 0.15,
    "masking_prob": 0.8,
    "batch_size": 2,
    "random_token_prob": 0.0,
    "dropout_rate": 0.1,
    "num_hosts": 7,
    "server_address": "3303428-worker-0:1234",
    "alphabet_size": 4105,
    "pad_token_id": 1,
    "mask_token_id": 2,
    "class_token_id": 3,
    "eos_token_id": -1,
    "prepend_bos": true,
    "append_eos": false,
    "max_positions": 1000,
    "emb_layer_norm_before": false,
    "attention_heads": 20,
    "embed_dim": 2560,
    "ffn_embed_dim": 10240,
    "num_layers": 32,
    "token_dropout": true,
    "embed_scale": 1.0,
    "use_remat": false,
    "architecture": "Vanilla",
    "acc_batch_size": 9,
    "num_local_devices": 8,
    "num_global_devices": 56,
    "tokens_length": 1000,
    "cutoff_size": 6000,
    "overlap": 200,
    "n_ratio": 0.05,
    "mixed-precision": true,
    "model_num_parameters": "2547800585",
    "1000G": true,
    "Train on all chromosomes": true
}