Update app.py

app.py

@@ -1,21 +1,18 @@
 import re
-from nltk.corpus import stopwords
 import spacy
 from transformers import T5Tokenizer, T5ForConditionalGeneration, Trainer, TrainingArguments
 from datasets import Dataset
 from sklearn.model_selection import train_test_split
 from spacy.cli import download
 
-# Modeli indir
+# Spacy modelini indir ve yükle
 download("en_core_web_sm")
-
-# Spacy modelini yükle
-nlp = spacy.load("en_core_web_sm")  # İngilizce model
+nlp = spacy.load("en_core_web_sm")
 
 # Metni temizleme fonksiyonu
 def clean_text_for_education_with_spacy(text):
-    doc = nlp(text)  # Spacy ile metni işleyin
-    tokens = [token.text for token in doc if not token.is_stop and not token.is_punct]  # Stop words ve noktalama işaretlerini kaldırır
+    doc = nlp(text)
+    tokens = [token.text for token in doc if not token.is_stop and not token.is_punct]
     return " ".join(tokens)
 
 # Prompts okuma
@@ -33,8 +30,8 @@ def read_prompts(file_path):
 
 # Dataset hazırlama
 def prepare_data(input_texts, target_texts, tokenizer):
-    inputs = tokenizer(input_texts, max_length=512, truncation=True, padding="max_length")
-    targets = tokenizer(target_texts, max_length=512, truncation=True, padding="max_length")
+    inputs = tokenizer(input_texts, max_length=256, truncation=True, padding="max_length")
+    targets = tokenizer(target_texts, max_length=256, truncation=True, padding="max_length")
     return {"input_ids": inputs["input_ids"], "attention_mask": inputs["attention_mask"], "labels": targets["input_ids"]}
 
 # Paraphrasing fonksiyonu