Add scripts used to generate v1.1.

scripts-for-generating-v1.1/pt_fix.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+import argparse
+import torch
+import transformers
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input_path", type=str, help="Input directory")
+    parser.add_argument("output_path", type=str, help="Output directory")
+    args = parser.parse_args()
+
+    robeczech = transformers.AutoModelForMaskedLM.from_pretrained(args.input_path, add_pooling_layer=True)
+
+    unk_id, mask_id, new_vocab = 3, 51960, 51997
+
+    assert robeczech.roberta.embeddings.word_embeddings.weight is robeczech.lm_head.decoder.weight
+    assert robeczech.lm_head.bias is robeczech.lm_head.decoder.bias
+    for weight in [robeczech.roberta.embeddings.word_embeddings.weight, robeczech.lm_head.bias]: #, robeczech.lm_head.decoder.weight]:
+        original = weight.data
+        assert original.shape[0] == mask_id + 1, original.shape
+        weight.data = torch.zeros((new_vocab,) + original.shape[1:], dtype=original.dtype)
+        weight.data[:mask_id + 1] = original
+        for new_unk in [mask_id - 1] + list(range(mask_id + 1, new_vocab)):
+            weight.data[new_unk] = original[unk_id]
+
+    robeczech.save_pretrained(args.output_path)
+    robeczech.save_pretrained(args.output_path, safe_serialization=False)

scripts-for-generating-v1.1/tf_fix.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+import argparse
+import transformers
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input_path", type=str, help="Input directory")
+    parser.add_argument("output_path", type=str, help="Output directory")
+    args = parser.parse_args()
+
+    robeczech = transformers.TFAutoModelWithLMHead.from_pretrained(args.input_path, from_pt=True)
+    robeczech.save_pretrained(args.output_path)

scripts-for-generating-v1.1/tokenizer_fix.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+
+import transformers
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input_path", type=str, help="Input directory")
+    parser.add_argument("output_path", type=str, help="Output directory")
+    args = parser.parse_args()
+
+    # Fix vocab.json
+    def fix_vocab(vocab):
+        mask_id = 51960
+        unused = mask_id + 1
+        remapped = []
+        fixed_vocab = {}
+        for key, value in vocab.items():
+            if value == 3 and key != "[UNK]":
+                if key == "ĠĊ":
+                    fixed_vocab[key] = mask_id - 1
+                else:
+                    remapped.append((key, unused))
+                    unused += 1
+            else:
+                fixed_vocab[key] = value
+
+        for key, value in remapped:
+            fixed_vocab[key] = value
+
+        return fixed_vocab
+
+    with open(os.path.join(args.input_path, "vocab.json"), "r", encoding="utf-8") as vocab_file:
+        vocab = json.load(vocab_file)
+
+    fixed_vocab = fix_vocab(vocab)
+
+    with open(os.path.join(args.output_path, "vocab.json"), "w", encoding="utf-8") as vocab_file:
+        json.dump(fixed_vocab, vocab_file, ensure_ascii=False, indent=None)
+        print(file=vocab_file)
+
+    # Regenerate tokenizer.json
+    tokenizer = transformers.AutoTokenizer.from_pretrained(args.output_path)
+    tokenizer._tokenizer.save(os.path.join(args.output_path, "tokenizer.json"))