File size: 3,017 Bytes
c6bb4cd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 |
from pdfminer.high_level import extract_text
import re
bentham_texts = []
import glob
def extract_text_from_pdf(pdf_path):
text = extract_text(pdf_path)
return text
bentham_pdfs = glob.glob('./Bentham*.pdf')
for pdf in bentham_pdfs:
print(pdf)
with open(pdf, 'rb') as f:
text = extract_text_from_pdf(f)
bentham_texts.append(text)
bentham_text_string = ' '.join(bentham_texts)
with open('bentham_text.txt', 'w') as f:
f.write(bentham_text_string)
##
bentham_text_string =''
with open('bentham_text.txt', 'r') as f:
bentham_text_strings = f.readlines()
bentham_text_string = ''.join(bentham_text_strings)
import re
def clean_text(text):
cleaned_text = re.sub(r'§\s*\d+\.', '', text)
# Step 2: Remove the unwanted patterns
# Removing patterns like "PLL v6.0 (generated September, 2011)" and URLs
cleaned_text = re.sub(r'\n*PLL v[0-9.]+ \(generated.*?\)\n+.*?\n+http.*?\n.*?Online Library of Liberty:.*?\n', '', cleaned_text, flags=re.DOTALL)
cleaned_text = re.sub(r'\n*\s*PLL v[0-9.]+ \(generated.*?\)\s*\n', '', cleaned_text, flags=re.DOTALL)
cleaned_text = re.sub(r'https?://\S+', '', cleaned_text)
# Removing "Online Library of Liberty" lines that might not fit the exact previous pattern
cleaned_text = re.sub(r'Online Library of Liberty:.*?\n', '', cleaned_text, flags=re.DOTALL)
cleaned_text = re.sub(r'\n\nPLL v[0-9.]+ \(generated.*?\)\n\n.*?\n\nhttp.*?\n', '', cleaned_text, flags=re.DOTALL)
cleaned_text = re.sub(r'[\x00-\x1F\x7F-\x9F]', '', cleaned_text) # Removes non-printable characters
cleaned_text = re.sub(r'\\[ntr]', '', cleaned_text) # Removes escaped special characters like \n, \t, \r
patterns_to_remove = [
r'^\s*$', # Empty lines
r'^\s*\d+\s*$', # Standalone numeric lines
r'\[Back to Table of Contents\]', # Specific placeholders
]
for pattern in patterns_to_remove:
cleaned_text = re.sub(pattern, '', cleaned_text, flags=re.MULTILINE)
return cleaned_text
cleaned_lines = []
for line in bentham_text_strings:
cleaned_line = clean_text(line)
if cleaned_line != '':
cleaned_lines.append(cleaned_line)
def split_into_chunks(text, chunk_size=100):
"""
Split the text into chunks of approximately `chunk_size` words.
Args:
text (str): The input text to split.
chunk_size (int): The desired chunk size in words.
Returns:
list of str: A list of text chunks.
"""
# Split the text into words
words = text.split()
# Create chunks of approximately `chunk_size` words
chunks = [' '.join(words[i:i + chunk_size]) for i in range(0, len(words), chunk_size)]
return chunks
chunks = split_into_chunks((' ').join(cleaned_lines), 100)
from datasets import Dataset
# Assuming `chunks` is the list of text chunks you created earlier
data = {'text': chunks}
new_dataset = Dataset.from_dict(data)
new_dataset.save_to_disk('./bentham_chunked')
|