add V2 models

Geneformer-V1-10M/config.json

@@ -0,0 +1,23 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.02,
+  "gradient_checkpointing": false,
+  "hidden_act": "relu",
+  "hidden_dropout_prob": 0.02,
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 2048,
+  "model_type": "bert",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.6.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 25426
+}

Geneformer-V1-10M/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5e33a757431643b3697de7ef6127950cdc49e06e58d4266b3a3ab191b683f14
+size 41183536

Geneformer-V1-10M/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d860e2125884475dd42bc2cd9a0e60c60808a7351241e08f2154931ffc142da
+size 41216562

Geneformer-V1-10M/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0ec3459454205174c9d2e4d6c6930f6b0fbf3364fc03a6f4d99c4d3add2012b
+size 2607

Geneformer-V2-104M/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "relu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 4096,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 20275
+}

Geneformer-V2-104M/generation_config.json

@@ -0,0 +1,5 @@
+{
+  "_from_model_config": true,
+  "pad_token_id": 0,
+  "transformers_version": "4.44.2"
+}

Geneformer-V2-104M/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fff5cba29ddd8792991fa77b4872246fbe548a178cebda3775cdc72b67780e7f
+size 417571156

Geneformer-V2-104M/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d8ddd9e4f35b5fe23a3adaae03aa4480705ca82eed546a488f970adb3752d9d
+size 5496

Geneformer-V2-104M_CLcancer/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "/gladstone/theodoris/lab/ctheodoris/gf-104m/models/241127_143148_geneformer_94M_L12_emb768_SL4096_E3_B18_LR0.0002_LScosine_WR0.007_Oadamw_DS13/models",
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "relu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 4096,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 20275
+}

Geneformer-V2-104M_CLcancer/generation_config.json

@@ -0,0 +1,5 @@
+{
+  "_from_model_config": true,
+  "pad_token_id": 0,
+  "transformers_version": "4.37.1"
+}

Geneformer-V2-104M_CLcancer/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:827738139bfed4bafa9d1f3df7c6146da2e3b85f7225076adc32c6eda0ba4357
+size 417571156

Geneformer-V2-104M_CLcancer/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cf8ce52b498253adc6df53197a99821fa145c19b8ae5eeb8d15be76b8b7ddb3
+size 4984

Geneformer-V2-316M/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "relu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1152,
+  "initializer_range": 0.02,
+  "intermediate_size": 4608,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 4096,
+  "model_type": "bert",
+  "num_attention_heads": 18,
+  "num_hidden_layers": 18,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 20275
+}

Geneformer-V2-316M/generation_config.json

@@ -0,0 +1,5 @@
+{
+  "_from_model_config": true,
+  "pad_token_id": 0,
+  "transformers_version": "4.44.2"
+}

Geneformer-V2-316M/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:965ceccea81953d362081ef3843560a0e4fef88d396c28017881f1e94b1246f3
+size 1265455076

Geneformer-V2-316M/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e45150f9a4ca34cb4e91ce79f65f3d99d9d66df9f66a37517a352d291008e0b8
+size 5432

fine_tuned_models/Geneformer-V1-10M_CellClassifier_cardiomyopathies_220224/config.json

@@ -0,0 +1,35 @@
+{
+  "_name_or_path": "/n/home01/ctheodoris/models/210602_111318_geneformer_27M_L6_emb256_SL2048_E3_B12_LR0.001_LSlinear_WU10000_Oadamw_DS12/models/",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.02,
+  "gradient_checkpointing": false,
+  "hidden_act": "relu",
+  "hidden_dropout_prob": 0.02,
+  "hidden_size": 256,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 2048,
+  "model_type": "bert",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "transformers_version": "4.6.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 25426
+}

fine_tuned_models/Geneformer-V1-10M_CellClassifier_cardiomyopathies_220224/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ced328122d57a847fc3914732337674500e259a82e64437c67b4954ac2f4e07
+size 73720721

fine_tuned_models/Geneformer-V1-10M_CellClassifier_cardiomyopathies_220224/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12ead3bad8cf4b853bac87eadeb79c9308ae492e9d29f32da1a2c85e8586108d
+size 41115113

fine_tuned_models/Geneformer-V1-10M_CellClassifier_cardiomyopathies_220224/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd8c0a739c2fe6a9ab4bb8f4a62ad8d7b879efcdceb5376b128a2040ff1bbe62
+size 14657

fine_tuned_models/Geneformer-V1-10M_CellClassifier_cardiomyopathies_220224/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d0797845afdae765a74ddab7966e0e1837617fd8171af8ee6aef9dedce248f2
+size 623

fine_tuned_models/Geneformer-V1-10M_CellClassifier_cardiomyopathies_220224/trainer_state.json

@@ -0,0 +1,150 @@
+{
+  "best_metric": 0.39658036828041077,
+  "best_model_checkpoint": "/n/holyscratch01/xiaoleliu_lab/Users/ctheodoris/models/220224_geneformer_27M_SequenceClassifier_tuning_hCMdCM_L2048_B12_LR1e-05_LScosine_WU500_E1_Oadamw_F2/run-8429a330/checkpoint-7020",
+  "epoch": 0.9,
+  "global_step": 7020,
+  "is_hyper_param_search": true,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00034606438343856935,
+      "loss": 0.911,
+      "step": 780
+    },
+    {
+      "epoch": 0.1,
+      "eval_accuracy": 0.4531576503366612,
+      "eval_loss": 1.4550466537475586,
+      "eval_runtime": 66.5164,
+      "eval_samples_per_second": 259.004,
+      "step": 780
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0006921287668771387,
+      "loss": 0.6273,
+      "step": 1560
+    },
+    {
+      "epoch": 0.2,
+      "eval_accuracy": 0.5953680055723242,
+      "eval_loss": 0.846651554107666,
+      "eval_runtime": 66.1267,
+      "eval_samples_per_second": 260.53,
+      "step": 1560
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0007330550166223805,
+      "loss": 0.5592,
+      "step": 2340
+    },
+    {
+      "epoch": 0.3,
+      "eval_accuracy": 0.5935105641978176,
+      "eval_loss": 1.0599186420440674,
+      "eval_runtime": 66.2608,
+      "eval_samples_per_second": 260.003,
+      "step": 2340
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0006283471571048975,
+      "loss": 0.3714,
+      "step": 3120
+    },
+    {
+      "epoch": 0.4,
+      "eval_accuracy": 0.686324587880195,
+      "eval_loss": 1.184874415397644,
+      "eval_runtime": 66.1411,
+      "eval_samples_per_second": 260.473,
+      "step": 3120
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0005236392975874146,
+      "loss": 0.2976,
+      "step": 3900
+    },
+    {
+      "epoch": 0.5,
+      "eval_accuracy": 0.7681100534014396,
+      "eval_loss": 0.6318939328193665,
+      "eval_runtime": 66.3309,
+      "eval_samples_per_second": 259.728,
+      "step": 3900
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0004189314380699318,
+      "loss": 0.2564,
+      "step": 4680
+    },
+    {
+      "epoch": 0.6,
+      "eval_accuracy": 0.7807058277223126,
+      "eval_loss": 0.7283642888069153,
+      "eval_runtime": 66.3416,
+      "eval_samples_per_second": 259.686,
+      "step": 4680
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0003142235785524487,
+      "loss": 0.2336,
+      "step": 5460
+    },
+    {
+      "epoch": 0.7,
+      "eval_accuracy": 0.8563965637334572,
+      "eval_loss": 0.5184123516082764,
+      "eval_runtime": 66.3416,
+      "eval_samples_per_second": 259.686,
+      "step": 5460
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0002095157190349659,
+      "loss": 0.1731,
+      "step": 6240
+    },
+    {
+      "epoch": 0.8,
+      "eval_accuracy": 0.8288832133735778,
+      "eval_loss": 0.5823884010314941,
+      "eval_runtime": 66.1535,
+      "eval_samples_per_second": 260.425,
+      "step": 6240
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 0.00010480785951748295,
+      "loss": 0.1451,
+      "step": 7020
+    },
+    {
+      "epoch": 0.9,
+      "eval_accuracy": 0.886812166241003,
+      "eval_loss": 0.39658036828041077,
+      "eval_runtime": 66.3555,
+      "eval_samples_per_second": 259.632,
+      "step": 7020
+    }
+  ],
+  "max_steps": 7800,
+  "num_train_epochs": 1,
+  "total_flos": 0,
+  "trial_name": null,
+  "trial_params": {
+    "learning_rate": 0.0008039341830649843,
+    "lr_scheduler_type": "polynomial",
+    "num_train_epochs": 1,
+    "per_device_train_batch_size": 12,
+    "seed": 73.15243080311434,
+    "warmup_steps": 1812.6785581609881,
+    "weight_decay": 0.2588277764570262
+  }
+}

fine_tuned_models/Geneformer-V1-10M_CellClassifier_cardiomyopathies_220224/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ffee119596c99b50a422b2f80103f4c44f7e25c2ea0e457fe224bad59f1f955
+size 2607

geneformer/ensembl_mapping_dict_gc104M.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0819bcbd869cfa14279449b037eb9ed1d09a91310e77bd1a19d927465030e95c
+size 3957652

geneformer/gene_median_dictionary_gc104M.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a51c53f6a771d64508dfaf61529df70e394c53bd20856926117ae5d641a24bf5
+size 1512661

geneformer/gene_name_id_dict_gc104M.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fabfa0c2f49c598c59ae432a32c3499a5908c033756c663b5e0cddf58deea8e1
+size 1660882

geneformer/token_dictionary_gc104M.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67c445f4385127adfc48dcc072320cd65d6822829bf27dd38070e6e787bc597f
+size 425590