Update fine_tuning.py

fine_tuning.py

@@ -3,24 +3,27 @@ from transformers import T5Tokenizer, T5ForConditionalGeneration, Trainer, Train
 from datasets import Dataset
 from sklearn.model_selection import train_test_split
 import re
-from nltk.corpus import stopwords
-from nltk.tokenize import word_tokenize
-from nltk.stem import PorterStemmer
 
 # Logging Ayarları
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 logger = logging.getLogger(__name__)
 
-
-stop_words = set(stopwords.words('english'))
-ps = PorterStemmer()
+stop_words = {"and", "or", "but", "the", "is", "are", "was", "were", "a", "an", "in", "on", "at", "of", "to", "with"}  # Örnek stop words
+def stem_word(word):
+    """PorterStemmer yerine basit bir gövdeleme fonksiyonu."""
+    suffixes = ['ing', 'ed', 'ly', 's', 'es', 'er']
+    for suffix in suffixes:
+        if word.endswith(suffix):
+            return word[:-len(suffix)]
+    return word
 
 def clean_text(text):
-    text = re.sub(r'[^\w\s]', '', text)
-    text = re.sub(r'\d+', '', text)
-    text = text.lower()
-    text = " ".join([word for word in text.split() if word not in stop_words])
-    text = " ".join([ps.stem(word) for word in word_tokenize(text)])
+    """Metin temizleme fonksiyonu."""
+    text = re.sub(r'[^\w\s]', '', text)  # Noktalama işaretlerini kaldır
+    text = re.sub(r'\d+', '', text)  # Sayıları kaldır
+    text = text.lower()  # Küçük harfe çevir
+    text = " ".join([word for word in text.split() if word not in stop_words])  # Stop words kaldır
+    text = " ".join([stem_word(word) for word in text.split()])  # Gövdeleme
     return text
 
 def read_prompts(file_path):
@@ -46,59 +49,21 @@ def paraphrase_with_model(text, model, tokenizer):
     output_ids = model.generate(
         inputs["input_ids"],
         do_sample=True,
-        top_k=40,  
-        top_p=0.9, 
-        temperature=0.8,  
-        max_length=200, 
-        no_repeat_ngram_size=3, 
+        top_k=40,
+        top_p=0.9,
+        temperature=0.8,
+        max_length=200,
+        no_repeat_ngram_size=3,
         early_stopping=True
     )
     return tokenizer.decode(output_ids[0], skip_special_tokens=True)
 
+# Model ve Tokenizer Yükleme
 model_name = "t5-base"
 tokenizer = T5Tokenizer.from_pretrained(model_name)
 model = T5ForConditionalGeneration.from_pretrained(model_name)
 
-input_texts, target_texts = read_prompts("prompts.txt")
-input_texts_cleaned = [clean_text(text) for text in input_texts]
-target_texts_cleaned = [clean_text(text) for text in target_texts]
-
-train_texts, val_texts, train_labels, val_labels = train_test_split(input_texts_cleaned, target_texts_cleaned, test_size=0.1)
-
-augmented_input_texts = input_texts_cleaned[:10] + [paraphrase_with_model(text, model, tokenizer) for text in input_texts_cleaned[:10]]
-augmented_target_texts = target_texts_cleaned[:10] + [paraphrase_with_model(text, model, tokenizer) for text in target_texts_cleaned[:10]]
-train_dataset = Dataset.from_dict(prepare_data(augmented_input_texts, augmented_target_texts))
-val_dataset = Dataset.from_dict(prepare_data(val_texts, val_labels))
-
-training_args = TrainingArguments(
-    output_dir="./results",
-    evaluation_strategy="steps",
-    learning_rate=5e-5,
-    per_device_train_batch_size=4,
-    num_train_epochs=3,
-    save_steps=500,
-    logging_dir="./logs",
-    logging_steps=10
-)
-
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=train_dataset,
-    eval_dataset=val_dataset
-)
-
-trainer.train()
-
-model.save_pretrained("./fine_tuned_model")
-tokenizer.save_pretrained("./fine_tuned_model")
-
 try:
-    logger.info("Loading tokenizer and model.")
-    model_name = "t5-base"
-    tokenizer = T5Tokenizer.from_pretrained(model_name)
-    model = T5ForConditionalGeneration.from_pretrained(model_name)
-
     logger.info("Reading and cleaning prompts.")
     input_texts, target_texts = read_prompts("prompts.txt")
     input_texts_cleaned = [clean_text(text) for text in input_texts]
@@ -111,6 +76,17 @@ try:
     train_dataset = Dataset.from_dict(prepare_data(train_texts, train_labels))
     val_dataset = Dataset.from_dict(prepare_data(val_texts, val_labels))
 
+    training_args = TrainingArguments(
+        output_dir="./results",
+        evaluation_strategy="steps",
+        learning_rate=5e-5,
+        per_device_train_batch_size=4,
+        num_train_epochs=3,
+        save_steps=500,
+        logging_dir="./logs",
+        logging_steps=10
+    )
+
     logger.info("Starting model training.")
     trainer = Trainer(
         model=model,