process_wake.py

import re
def is_page_number(line):
    return line.strip().isdigit()
with open("./finn_wake.txt", "r", encoding="utf-8") as file:
    lines = file.readlines()
filtered_lines = [line for line in lines if not is_page_number(line)]
text = ''.join(filtered_lines)
from datasets import Dataset
import pandas as pd
def split_paragraph_into_smaller_parts(paragraph, max_length=100):
    """Split a paragraph into smaller parts with a maximum length in words."""
    words = paragraph.split()
    for i in range(0, len(words), max_length):
        yield ' '.join(words[i:i+max_length])
paragraphs = text.split('\n')
split_paragraphs = []
for paragraph in paragraphs:
    if paragraph.strip() != "":
        split_paragraphs.extend(split_paragraph_into_smaller_parts(paragraph, max_length=100))
df = pd.DataFrame(split_paragraphs, columns=['text'])
dataset = Dataset.from_pandas(df)
df.to_csv('finn_wake.csv', index=False)
dataset.save_to_disk('finn_wake_dataset')