Update fine_tuning.py

fine_tuning.py

@@ -1,66 +1,158 @@
 from transformers import T5Tokenizer, T5ForConditionalGeneration, Trainer, TrainingArguments
 from datasets import Dataset
 from sklearn.model_selection import train_test_split
+import random
+import re
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+from nltk.stem import PorterStemmer
 
-# Load model and tokenizer
-model_name = "t5-base"
-tokenizer = T5Tokenizer.from_pretrained(model_name)
-model = T5ForConditionalGeneration.from_pretrained(model_name)
+def clean_pdf_text(pdf_text):
+    # Satır başlarındaki ve sonlarındaki boşlukları temizleyelim
+    pdf_text = pdf_text.strip()
+    
+    # Gereksiz satır aralıklarını kaldırma
+    pdf_text = re.sub(r'\n+', ' ', pdf_text)
+    
+    # Sayfa numarası gibi gereksiz kısımları kaldırma (örneğin 'Page 1', 'Page 2' gibi)
+    pdf_text = re.sub(r'\bPage \d+\b', '', pdf_text)
+    
+    return pdf_text
+
+def pdf_to_text(pdf_path):
+    """ Converts PDF to text """
+    pdf_text = extract_text(pdf_path)
+    return clean_pdf_text(pdf_text)
+
+# Stop words
+stop_words = set(stopwords.words('english'))
+
+# Stemmer
+ps = PorterStemmer()
 
-# Prepare the dataset for training
-def prepare_data(input_texts, target_texts, tokenizer):
-    inputs = tokenizer(input_texts, max_length=512, truncation=True, padding="max_length")
-    targets = tokenizer(target_texts, max_length=512, truncation=True, padding="max_length")
-    return {"input_ids": inputs["input_ids"], "attention_mask": inputs["attention_mask"], "labels": targets["input_ids"]}
+# Metni temizleme fonksiyonu
+def clean_text(text):
+    # Noktalama işaretlerini kaldırma
+    text = re.sub(r'[^\w\s]', '', text)
+    
+    # Sayıları kaldırma
+    text = re.sub(r'\d+', '', text)
+    
+    # Küçük harfe çevirme
+    text = text.lower()
+
+    # Stop words kaldırma
+    text = " ".join([word for word in text.split() if word not in stop_words])
+    
+    # Kelimeleri köklerine indirgeme (stemming)
+    text = " ".join([ps.stem(word) for word in word_tokenize(text)])
+    
+    return text
 
-# Paraphrasing function
+# Paraphrasing fonksiyonu
 def paraphrase_with_model(text, model, tokenizer):
-    prompt = "Teach the following content: " + text
+    prompt = "paraphrase: " + text  # T5 modeline paraphrasing görevi verdiğimizi belirtiyoruz.
+    
+    # Tokenizer ile metni tokenlara dönüştürme
     inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True, max_length=512)
+    
+    # Sampling yöntemi ile model çalıştırma
     output_ids = model.generate(
         inputs["input_ids"],
-        do_sample=False,  # For deterministic results
-        max_length=150,
-        no_repeat_ngram_size=2,
-        early_stopping=True
+        do_sample=True,  # Sampling modunu aktif hale getirme
+        top_k=50,  # Top-k sampling
+        top_p=0.95,  # Top-p sampling
+        temperature=1.0,  # Daha fazla çeşitlilik için temperature
+        max_length=150,  # Maksimum cümle uzunluğu
+        no_repeat_ngram_size=2,  # Aynı n-gramların tekrarını engelle
+        early_stopping=True  # Daha erken durdurma
     )
-    return tokenizer.decode(output_ids[0], skip_special_tokens=True)
+    
+    # Modelin çıktısını decode ederek metni çözme, maksimum uzunluğu sınırlandırın
+    paraphrased_text = tokenizer.decode(output_ids[0], skip_special_tokens=True, max_length=150)
 
-# Fine-tuning function
-def fine_tune_model(input_texts, target_texts):
-    # Split data into training and validation sets
-    train_texts, val_texts, train_labels, val_labels = train_test_split(input_texts, target_texts, test_size=0.1)
+    
+    return paraphrased_text
+
+# Tokenizer ve modelin yüklenmesi
+model_name = "google-t5/t5-base"
+tokenizer = T5Tokenizer.from_pretrained(model_name)
+model = T5ForConditionalGeneration.from_pretrained(model_name)
 
-    # Data augmentation with paraphrasing
-    augmented_input_texts = input_texts + [paraphrase_with_model(text, model, tokenizer) for text in input_texts[:10]]
-    augmented_target_texts = target_texts + [paraphrase_with_model(text, model, tokenizer) for text in target_texts[:10]]
+def prepare_data(input_texts, target_texts):
+    inputs = tokenizer(input_texts, max_length=512, truncation=True, padding=True, return_tensors="pt")
+    targets = tokenizer(target_texts, max_length=512, truncation=True, padding=True, return_tensors="pt")
     
-    train_dataset = Dataset.from_dict(prepare_data(augmented_input_texts, augmented_target_texts, tokenizer))
-    val_dataset = Dataset.from_dict(prepare_data(val_texts, val_labels, tokenizer))
-
-    # Training arguments
-    training_args = TrainingArguments(
-        output_dir="./results",
-        evaluation_strategy="steps",
-        learning_rate=5e-5,
-        per_device_train_batch_size=4,
-        num_train_epochs=3,
-        save_steps=500,
-        logging_dir="./logs",
-        logging_steps=10
-    )
+    # Labels olarak hedef metinleri ayarlıyoruz
+    inputs["labels"] = targets["input_ids"]
+    
+    # Attention maskeleri de dahil et
+    inputs["attention_mask"] = inputs["attention_mask"]
+    
+    return inputs
 
-    # Trainer setup
-    trainer = Trainer(
-        model=model,
-        args=training_args,
-        train_dataset=train_dataset,
-        eval_dataset=val_dataset
-    )
 
-    # Training
-    trainer.train()
+# Eğitim verileri
+input_texts = [ 
+    "Site Reliability Engineering is a concept born at Google. Ben Trainor's team of seven people started it in 2003 to keep Google.com running reliably.",
+    "Reliability is critical for any system. Without reliability, even the best features are useless as users can't access them.",
+    "SRE teams at Google handle large-scale systems with efficiency, working closely with developers to ensure scalability, reliability, and cost-effectiveness.",
+    "Site Reliability Engineering treats operations as a software engineering problem, making it distinct from traditional operations teams."]
+target_texts = [ 
+    "SRE was introduced at Google in 2003 by Ben Trainor's team to ensure the reliability of Google.com.",
+    "Reliability is essential for a system to be usable; without it, features lose value.",
+    "Google's SRE teams collaborate with developers to manage large-scale systems efficiently and reliably.",
+    "SRE approaches operations as a software engineering task, revolutionizing traditional operational methods."]
+
+# Veriyi temizleme
+input_texts_cleaned = [clean_text(text) for text in input_texts]
+target_texts_cleaned = [clean_text(text) for text in target_texts]
+
+# Eğitim ve doğrulama verisini ayırma
+train_texts, val_texts, train_labels, val_labels = train_test_split(input_texts_cleaned, target_texts_cleaned, test_size=0.1)
+
+# Eğitim ve doğrulama verilerini hazırlama
+train_dataset = Dataset.from_dict(prepare_data(train_texts, train_labels))
+val_dataset = Dataset.from_dict(prepare_data(val_texts, val_labels))
+
+# Eğitim verisini augment etmek (opsiyonel)
+augmented_input_texts = [paraphrase_with_model(text, model, tokenizer) for text in input_texts_cleaned[:10]]  # Daha küçük bir örnekle başla
+augmented_target_texts = [paraphrase_with_model(text, model, tokenizer) for text in target_texts_cleaned[:10]]  # Aynı şekilde
+augmented_dataset = prepare_data(augmented_input_texts, augmented_target_texts)
+train_dataset = Dataset.from_dict(augmented_dataset)
+
+# Eğitim argümanları
+training_args = TrainingArguments(
+    output_dir="./results",
+    evaluation_strategy="steps",
+    learning_rate=5e-5,
+    per_device_train_batch_size=4,
+    per_device_eval_batch_size=4,
+    num_train_epochs=3,
+    weight_decay=0.01,
+    save_steps=500,  # Modeli her 500 adımda bir kaydet
+    eval_steps=500,  # Değerlendirmeyi her 500 adımda bir yap
+    save_total_limit=2,
+    logging_dir="./logs",
+    logging_steps=10,
+    load_best_model_at_end=True
+)
+
+# Trainer tanımı
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=val_dataset  # Değerlendirme kümesi eklenmeli
+)
+
+# Fine-tuning başlatma
+trainer.train()
+
+# Fine-tuned modelin kaydedilmesi
+model.save_pretrained("./fine_tuned_model")
+tokenizer.save_pretrained("./fine_tuned_model")
 
-    # Save the fine-tuned model
-    model.save_pretrained("./fine_tuned_model")
-    tokenizer.save_pretrained("./fine_tuned_model")
+# Fine-tuned modelinizi yükleyin
+model = T5ForConditionalGeneration.from_pretrained("./fine_tuned_model")
+tokenizer = T5Tokenizer.from_pretrained("./fine_tuned_model")