Upload 41 files

fine_tune_joyce.py

@@ -0,0 +1,47 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments
+from datasets import load_dataset, load_from_disk
+
+dataset = load_from_disk('finn_wake_dataset')
+
+tokenizer = AutoTokenizer.from_pretrained("tinyllama/tinyllama-1.1b-chat-v1.0")
+
+tokenizer.save_pretrained(".results/checkpoint-12000/")
+
+model = AutoModelForCausalLM.from_pretrained("tinyllama/tinyllama-1.1b-chat-v1.0")
+
+if tokenizer.pad_token is None:
+    print("Tokenizer does not have a pad token set. Setting pad_token to eos_token.")
+    tokenizer.pad_token = tokenizer.eos_token
+
+def tokenize_function(examples):
+    
+    tokenized_inputs = tokenizer(examples['text'], padding="max_length", truncation=True, max_length=128)
+    tokenized_inputs["labels"] = tokenized_inputs["input_ids"].copy()
+    
+    return  tokenized_inputs
+    
+tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=["text"])
+train_test_split = tokenized_dataset.train_test_split(test_size=0.1)  
+
+train_dataset = train_test_split['train']
+eval_dataset = train_test_split['test']
+training_args = TrainingArguments(
+    output_dir="./results",
+    num_train_epochs=3, 
+    per_device_train_batch_size=1,  
+    warmup_steps=500,
+    weight_decay=0.01,
+    logging_dir="./logs",
+    logging_steps=10,
+    save_strategy="steps",
+    save_steps=500,
+    save_total_limit=2,
+    use_cpu=True)
+trainer = Trainer(
+    model=model,  
+    args=training_args,
+    train_dataset=train_dataset, 
+    eval_dataset=eval_dataset, 
+)
+#below has been modified because i ran out of disk storage initially so had to resume and adjust the save_strategy above.
+trainer.train(resume_from_checkpoint="./results/checkpoint-10000")

finn_wake_dataset/cache-07333d095ed3feb5.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7799f52711777e954892b9143657a98eb4ad4412c8eb9ec8d51cf9f2d29d0960
+size 33738392

finn_wake_dataset/cache-08eaee58c5a946a5.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57d59e9370cb6130a4315fa2e751c32aab502b2f9bf80f25cb52e243364f6915
+size 23368

finn_wake_dataset/cache-108bb49fa40e19cc.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fee0801181eb75905ceec32c2100b5e957535fec236e46f6268fb204b9a87f2d
+size 23368

finn_wake_dataset/cache-21a914d68d7decd3.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4db4f65eb28996e319333d5b010d0881b9a6095fc04f8fd9c6eba422037813d5
+size 23368

finn_wake_dataset/cache-21efcd3d18f14eb8.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:751aada291665431650955366b4430502c33b9b2c2aaf7376eaf89abb08e9621
+size 83745984

finn_wake_dataset/cache-25107278423e0b27.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9866ffc0fbbc4c8bc780314aa2fcc9dabf43df74f71d3944b67f37ab0e794158
+size 2968

finn_wake_dataset/cache-294de0f58e07010c.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4af4228221d06dc079633bac9bf6f84af52752e3b569e99b5da1a4283e1c8ce4
+size 33832

finn_wake_dataset/cache-3d5c944f43725d69.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ac4e76ce5d8ba937fd8ab9db955eacb62c1d2705c5a5b9765253687549e3e33
+size 2968

finn_wake_dataset/cache-40ed1615346dc08d.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f10c74d395f429b9b9795271ac556ede1daf8b8c05f04986d7f54fb4aea0688
+size 23368

finn_wake_dataset/cache-4160687fb3073882.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ad2214bd7e550cec284ffaa766b916a076a0e75fe377ce855634dbfeea467ae
+size 23368

finn_wake_dataset/cache-4328f755c07fa7f2.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:549c73a76cd629ad9e36435289c0354fbd8c44dc91edcd3a9e3a2c8efa604e67
+size 33738392

finn_wake_dataset/cache-461df009004232f3.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d214cac2a17f55672d3c7a182a25beb09ac4a6232f91518449bf64b4c36e4aa6
+size 33832

finn_wake_dataset/cache-4a71b21bdeaa2ec7.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ec3d6cde0807e531ecf7b92decd38a27a3ace2c04f2190721abd9d4968d9255
+size 2968

finn_wake_dataset/cache-54251f52f75d36f0.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca4cd45ca17b5d83c8c97bea2fab7ceef28b99931062c92cb1e19bbf02c63415
+size 4096

finn_wake_dataset/cache-5929bf28441deefd.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:801a239d436b573afd7561832db31eaf9cc40c878a4bc6aeccbdf9913df699e9
+size 23368

finn_wake_dataset/cache-5be06efb6c2a7249.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a992fa9985e1e353bb8ade640fbfff72e2e75db5a2a72d8bec3ba5a258ba99c
+size 33832

finn_wake_dataset/cache-84a914c4d2cc8d93.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad6c49e3bdb4f56e85c5f62077186a62d17646acc4d8fc1440cfd4317e1278d9
+size 2968

finn_wake_dataset/cache-8cc3936385189eb8.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b077a811daba8e024ca0e53fcc08565f1f2d676e586f39b25f88881b45adc60
+size 4096

finn_wake_dataset/cache-8f8de0a505da9b57.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ace69e62138b3125ae070c00efafd2193e23e6cf52199a49523852169bca17d
+size 2968

finn_wake_dataset/cache-93f1332b83d2a02e.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a90517adc17bac71edd4db47e7fe7780687ba2a70711111cd43fd69a6592fe9
+size 4096

finn_wake_dataset/cache-a463b77c9c10fcdd.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b236cc0e50d9affe1746fc1530b3e99a9d572acce641e26d0c5ef4cb164af744
+size 2968

finn_wake_dataset/cache-b41c1b1ef780b910.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e26f42fdfc293b91e57c84890b675de30bbdb88238775d0addbbaf488e87335
+size 2968

finn_wake_dataset/cache-ba9518a20dfb972e.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3967fa682ccf74313b48cab8a799cde75a4dd48f7324cdb0361feef6d42d12eb
+size 23368

finn_wake_dataset/cache-dda16c8e4395755d.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe7a51ecd1cb6d50858465c20d310f58a8ca6428dc041313d4c84bf7ba333c74
+size 23368

finn_wake_dataset/cache-e1c4ec6e99052dfc.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:462b68fef59b237820edd0a402230fb1e3953f1e3572977cf15b2e0ad46400a3
+size 5271744

finn_wake_dataset/cache-e6f91d075c0b5063.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb7be6b6fbca4557816f0dd00b3ac3fcfbbe0a08a2c93b851455ae642b2e116a
+size 7640328

finn_wake_dataset/cache-e79c82020953bbef.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04731bd4add6cb26e32f9e697f5764f421f64e62e774b13371743ed48327f4c0
+size 23368

finn_wake_dataset/cache-f135f63843848cc4.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4a87a79fea2a4dac7a8607f5aab278894807282427da346899de8d52b132100
+size 2968

finn_wake_dataset/cache-f39a662b6194c6e4.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a591977c03ca3bde5d1916489df376e89bcce387d660394adfaf90448d5bc4a
+size 2968

finn_wake_dataset/cache-f7fe086af672c971.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36a242fdf45fcea8a806a8df21299049d99a2c6a3489a9a6ddfdab82626df1ef
+size 23368

finn_wake_dataset/cache-fb842fe1846f33ac.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb719489a9a6868c7f28d4e237e1d7ad516b74333fc2adff4dea6010d26e381b
+size 23368

finn_wake_dataset/cache-fda59bdb198b3ef4.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9bd80aa28367576aaa834a272d4a192d554d2fc3f70914d1b53db9851f04231
+size 2968

finn_wake_dataset/cache-fefdfd5378240940.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:867fbcea23ec00b23b23e36e1522a1795d6aa179f27d098fae2973fe3b3ba38c
+size 2968

finn_wake_dataset/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3684e77cfcc0fdcc1a627db2f5caa5f306f61f70ee4d2434f205293fc491037
+size 1321944

finn_wake_dataset/dataset_info.json

@@ -0,0 +1,12 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "text": {
+      "dtype": "string",
+      "_type": "Value"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

finn_wake_dataset/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "953e00a2598c0e70",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

process_wake.py

@@ -0,0 +1,23 @@
+import re
+def is_page_number(line):
+    return line.strip().isdigit()
+with open("./finn_wake.txt", "r", encoding="utf-8") as file:
+    lines = file.readlines()
+filtered_lines = [line for line in lines if not is_page_number(line)]
+text = ''.join(filtered_lines)
+from datasets import Dataset
+import pandas as pd
+def split_paragraph_into_smaller_parts(paragraph, max_length=100):
+    """Split a paragraph into smaller parts with a maximum length in words."""
+    words = paragraph.split()
+    for i in range(0, len(words), max_length):
+        yield ' '.join(words[i:i+max_length])
+paragraphs = text.split('\n')
+split_paragraphs = []
+for paragraph in paragraphs:
+    if paragraph.strip() != "":
+        split_paragraphs.extend(split_paragraph_into_smaller_parts(paragraph, max_length=100))
+df = pd.DataFrame(split_paragraphs, columns=['text'])
+dataset = Dataset.from_pandas(df)
+df.to_csv('finn_wake.csv', index=False)
+dataset.save_to_disk('finn_wake_dataset')

text_gen.py

@@ -0,0 +1,27 @@
+
+
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments
+from datasets import load_dataset, load_from_disk
+
+#post training
+model_path = "./results/checkpoint-12000" 
+model = AutoModelForCausalLM.from_pretrained(model_path)
+tokenizer = AutoTokenizer.from_pretrained("tinyllama/tinyllama-1.1b-chat-v1.0")
+
+
+input_text = "ae left to go to ireland and found a fairy"  
+input_ids = tokenizer.encode(input_text, return_tensors='pt')
+output = model.generate(
+    input_ids=tokenizer.encode(input_text, return_tensors="pt"),
+    max_length=400,
+    num_return_sequences=1, 
+    temperature=0.7,
+    top_k=50,  
+    top_p=0.95, 
+    do_sample=True,
+    num_beams=5
+)
+
+decoded_output = tokenizer.decode(output[0], skip_special_tokens=True)
+print(decoded_output)