Upload tokenizer

added_tokens.json

@@ -0,0 +1,7 @@
+{
+  "[CLS]": 28,
+  "[MASK]": 26,
+  "[PAD]": 25,
+  "[SEP]": 27,
+  "[UNK]": 24
+}

special_tokens_map.json

@@ -0,0 +1,25 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": "[SEP]",
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,95 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 24,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 25,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 26,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 27,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 28,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": []
+  },
+  "pre_tokenizer": {
+    "type": "WhitespaceSplit"
+  },
+  "post_processor": null,
+  "decoder": {
+    "type": "WordPiece",
+    "prefix": "##",
+    "cleanup": true
+  },
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "<boc>": 0,
+      "<eoc>": 1,
+      "<sepc>": 2,
+      "<bog>": 3,
+      "<eog>": 4,
+      "<sepg>": 5,
+      "IDX_0": 6,
+      "IDX_1": 7,
+      "IDX_2": 8,
+      "IDX_3": 9,
+      "IDX_4": 10,
+      "IDX_5": 11,
+      "IDX_6": 12,
+      "IDX_7": 13,
+      "IDX_8": 14,
+      "IDX_9": 15,
+      "ATOM_C": 16,
+      "ATOM_N": 17,
+      "ATOM_O": 18,
+      "ATOM_F": 19,
+      "BOND_SINGLE": 20,
+      "BOND_DOUBLE": 21,
+      "BOND_TRIPLE": 22,
+      "BOND_AROMATIC": 23
+    },
+    "unk_token": "[UNK]"
+  }
+}

tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "24": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "25": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "26": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "27": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 85,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

vocab.json

@@ -0,0 +1 @@
+{"<boc>":0,"<eoc>":1,"<sepc>":2,"<bog>":3,"<eog>":4,"<sepg>":5,"IDX_0":6,"IDX_1":7,"IDX_2":8,"IDX_3":9,"IDX_4":10,"IDX_5":11,"IDX_6":12,"IDX_7":13,"IDX_8":14,"IDX_9":15,"ATOM_C":16,"ATOM_N":17,"ATOM_O":18,"ATOM_F":19,"BOND_SINGLE":20,"BOND_DOUBLE":21,"BOND_TRIPLE":22,"BOND_AROMATIC":23}